2024-11-15 17:39:36,698 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 17:39:36,713 main DEBUG Took 0.012967 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 17:39:36,714 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 17:39:36,715 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 17:39:36,716 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 17:39:36,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,736 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 17:39:36,757 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,759 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,760 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,760 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,761 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,762 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,763 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,764 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,765 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,765 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,767 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,767 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,768 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,768 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,769 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,770 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,770 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,771 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,772 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,772 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,773 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,774 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,774 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,775 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:39:36,776 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,776 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 17:39:36,778 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:39:36,780 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 17:39:36,783 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 17:39:36,784 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 17:39:36,786 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 17:39:36,786 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 17:39:36,798 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 17:39:36,801 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 17:39:36,804 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 17:39:36,805 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 17:39:36,805 main DEBUG createAppenders(={Console}) 2024-11-15 17:39:36,806 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 17:39:36,807 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 17:39:36,807 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 17:39:36,811 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 17:39:36,811 main DEBUG OutputStream closed 2024-11-15 17:39:36,812 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 17:39:36,812 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 17:39:36,812 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 17:39:36,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 17:39:36,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 17:39:36,923 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 17:39:36,924 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 17:39:36,924 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 17:39:36,925 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 17:39:36,926 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 17:39:36,926 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 17:39:36,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 17:39:36,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 17:39:36,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 17:39:36,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 17:39:36,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 17:39:36,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 17:39:36,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 17:39:36,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 17:39:36,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 17:39:36,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 17:39:36,934 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 17:39:36,934 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 17:39:36,934 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 17:39:36,935 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T17:39:37,336 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff 2024-11-15 17:39:37,339 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 17:39:37,339 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T17:39:37,351 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-15T17:39:37,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T17:39:37,402 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e, deleteOnExit=true 2024-11-15T17:39:37,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T17:39:37,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/test.cache.data in system properties and HBase conf 2024-11-15T17:39:37,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T17:39:37,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/hadoop.log.dir in system properties and HBase conf 2024-11-15T17:39:37,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T17:39:37,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T17:39:37,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T17:39:37,527 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T17:39:37,610 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T17:39:37,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T17:39:37,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T17:39:37,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T17:39:37,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T17:39:37,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T17:39:37,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T17:39:37,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T17:39:37,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T17:39:37,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T17:39:37,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/nfs.dump.dir in system properties and HBase conf 2024-11-15T17:39:37,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/java.io.tmpdir in system properties and HBase conf 2024-11-15T17:39:37,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T17:39:37,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T17:39:37,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T17:39:38,909 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T17:39:39,016 INFO [Time-limited test {}] log.Log(170): Logging initialized @3240ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T17:39:39,114 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T17:39:39,197 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T17:39:39,239 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T17:39:39,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T17:39:39,242 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T17:39:39,262 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T17:39:39,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/hadoop.log.dir/,AVAILABLE} 2024-11-15T17:39:39,268 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T17:39:39,480 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/java.io.tmpdir/jetty-localhost-41605-hadoop-hdfs-3_4_1-tests_jar-_-any-4609335346434929640/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T17:39:39,486 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41605} 2024-11-15T17:39:39,486 INFO [Time-limited test {}] server.Server(415): Started @3711ms 2024-11-15T17:39:40,090 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T17:39:40,098 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T17:39:40,099 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T17:39:40,099 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T17:39:40,099 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T17:39:40,100 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/hadoop.log.dir/,AVAILABLE} 2024-11-15T17:39:40,100 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T17:39:40,197 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/java.io.tmpdir/jetty-localhost-35939-hadoop-hdfs-3_4_1-tests_jar-_-any-3827270510908853789/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T17:39:40,197 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35939} 2024-11-15T17:39:40,197 INFO [Time-limited test {}] server.Server(415): Started @4423ms 2024-11-15T17:39:40,244 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T17:39:41,078 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/data/data2/current/BP-624044830-172.17.0.2-1731692378459/current, will proceed with Du for space computation calculation, 2024-11-15T17:39:41,078 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/data/data1/current/BP-624044830-172.17.0.2-1731692378459/current, will proceed with Du for space computation calculation, 2024-11-15T17:39:41,103 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T17:39:41,144 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x30de8d54aa837202 with lease ID 0x301b0f684a2260ec: Processing first storage report for DS-691194fc-f13b-4ebb-87e2-4599068e9e0a from datanode DatanodeRegistration(127.0.0.1:43115, datanodeUuid=cdf78b8e-f28b-455a-8346-40004e1fb9d6, infoPort=42963, infoSecurePort=0, ipcPort=35899, storageInfo=lv=-57;cid=testClusterID;nsid=169941397;c=1731692378459) 2024-11-15T17:39:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x30de8d54aa837202 with lease ID 0x301b0f684a2260ec: from storage DS-691194fc-f13b-4ebb-87e2-4599068e9e0a node DatanodeRegistration(127.0.0.1:43115, datanodeUuid=cdf78b8e-f28b-455a-8346-40004e1fb9d6, infoPort=42963, infoSecurePort=0, ipcPort=35899, storageInfo=lv=-57;cid=testClusterID;nsid=169941397;c=1731692378459), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-15T17:39:41,146 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x30de8d54aa837202 with lease ID 0x301b0f684a2260ec: Processing first storage report for DS-5d7ca6a1-be73-4b0d-8af1-36293f6d0d1f from datanode DatanodeRegistration(127.0.0.1:43115, datanodeUuid=cdf78b8e-f28b-455a-8346-40004e1fb9d6, infoPort=42963, infoSecurePort=0, ipcPort=35899, storageInfo=lv=-57;cid=testClusterID;nsid=169941397;c=1731692378459) 2024-11-15T17:39:41,146 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x30de8d54aa837202 with lease ID 0x301b0f684a2260ec: from storage DS-5d7ca6a1-be73-4b0d-8af1-36293f6d0d1f node DatanodeRegistration(127.0.0.1:43115, datanodeUuid=cdf78b8e-f28b-455a-8346-40004e1fb9d6, infoPort=42963, infoSecurePort=0, ipcPort=35899, storageInfo=lv=-57;cid=testClusterID;nsid=169941397;c=1731692378459), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T17:39:41,240 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff 2024-11-15T17:39:41,307 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/zookeeper_0, clientPort=61131, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T17:39:41,316 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61131 2024-11-15T17:39:41,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:39:41,330 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:39:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741825_1001 (size=7) 2024-11-15T17:39:41,940 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e with version=8 2024-11-15T17:39:41,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/hbase-staging 2024-11-15T17:39:42,024 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T17:39:42,327 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4e3b17656000:0 server-side Connection retries=45 2024-11-15T17:39:42,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:39:42,337 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T17:39:42,342 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T17:39:42,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:39:42,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T17:39:42,491 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T17:39:42,545 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T17:39:42,557 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T17:39:42,562 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T17:39:42,596 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 992 (auto-detected) 2024-11-15T17:39:42,597 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T17:39:42,623 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41205 2024-11-15T17:39:42,652 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41205 connecting to ZooKeeper ensemble=127.0.0.1:61131 2024-11-15T17:39:42,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412050x0, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T17:39:42,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41205-0x10140de572c0000 connected 2024-11-15T17:39:42,849 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:39:42,853 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:39:42,865 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:39:42,870 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e, hbase.cluster.distributed=false 2024-11-15T17:39:42,902 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T17:39:42,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41205 2024-11-15T17:39:42,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41205 2024-11-15T17:39:42,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41205 2024-11-15T17:39:42,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41205 2024-11-15T17:39:42,922 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41205 2024-11-15T17:39:43,039 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4e3b17656000:0 server-side Connection retries=45 2024-11-15T17:39:43,040 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:39:43,041 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T17:39:43,041 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T17:39:43,041 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:39:43,041 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T17:39:43,044 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T17:39:43,046 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T17:39:43,048 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45187 2024-11-15T17:39:43,049 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45187 connecting to ZooKeeper ensemble=127.0.0.1:61131 2024-11-15T17:39:43,051 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:39:43,054 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:39:43,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451870x0, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T17:39:43,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:451870x0, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:39:43,087 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45187-0x10140de572c0001 connected 2024-11-15T17:39:43,091 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T17:39:43,101 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T17:39:43,104 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T17:39:43,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T17:39:43,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45187 2024-11-15T17:39:43,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45187 2024-11-15T17:39:43,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45187 2024-11-15T17:39:43,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45187 2024-11-15T17:39:43,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45187 2024-11-15T17:39:43,131 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4e3b17656000:41205 2024-11-15T17:39:43,132 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4e3b17656000,41205,1731692382118 2024-11-15T17:39:43,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:39:43,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:39:43,149 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4e3b17656000,41205,1731692382118 2024-11-15T17:39:43,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T17:39:43,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:43,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:43,182 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T17:39:43,183 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4e3b17656000,41205,1731692382118 from backup master directory 2024-11-15T17:39:43,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4e3b17656000,41205,1731692382118 2024-11-15T17:39:43,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:39:43,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:39:43,195 WARN [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T17:39:43,195 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4e3b17656000,41205,1731692382118 2024-11-15T17:39:43,197 INFO [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T17:39:43,199 INFO [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T17:39:43,249 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/hbase.id] with ID: f848a115-bc41-4153-aa8f-4783b69d52ca 2024-11-15T17:39:43,250 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/.tmp/hbase.id 2024-11-15T17:39:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741826_1002 (size=42) 2024-11-15T17:39:43,665 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/.tmp/hbase.id]:[hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/hbase.id] 2024-11-15T17:39:43,716 INFO [master/4e3b17656000:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:39:43,721 INFO [master/4e3b17656000:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T17:39:43,738 INFO [master/4e3b17656000:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-15T17:39:43,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:43,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741827_1003 (size=196) 2024-11-15T17:39:44,266 INFO [master/4e3b17656000:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:44,269 INFO [master/4e3b17656000:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T17:39:44,284 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:44,288 INFO [master/4e3b17656000:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T17:39:44,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741828_1004 (size=1189) 2024-11-15T17:39:44,740 INFO [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store 2024-11-15T17:39:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741829_1005 (size=34) 2024-11-15T17:39:45,163 INFO [master/4e3b17656000:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T17:39:45,167 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:45,168 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T17:39:45,169 INFO [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:39:45,169 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:39:45,171 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T17:39:45,172 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:39:45,172 INFO [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:39:45,174 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731692385168Disabling compacts and flushes for region at 1731692385168Disabling writes for close at 1731692385172 (+4 ms)Writing region close event to WAL at 1731692385172Closed at 1731692385172 2024-11-15T17:39:45,177 WARN [master/4e3b17656000:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/.initializing 2024-11-15T17:39:45,177 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/WALs/4e3b17656000,41205,1731692382118 2024-11-15T17:39:45,189 INFO [master/4e3b17656000:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T17:39:45,206 INFO [master/4e3b17656000:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4e3b17656000%2C41205%2C1731692382118, suffix=, logDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/WALs/4e3b17656000,41205,1731692382118, archiveDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/oldWALs, maxLogs=10 2024-11-15T17:39:45,230 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/WALs/4e3b17656000,41205,1731692382118/4e3b17656000%2C41205%2C1731692382118.1731692385212, exclude list is [], retry=0 2024-11-15T17:39:45,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43115,DS-691194fc-f13b-4ebb-87e2-4599068e9e0a,DISK] 2024-11-15T17:39:45,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T17:39:45,292 INFO [master/4e3b17656000:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/WALs/4e3b17656000,41205,1731692382118/4e3b17656000%2C41205%2C1731692382118.1731692385212 2024-11-15T17:39:45,295 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42963:42963)] 2024-11-15T17:39:45,296 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:45,296 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:45,300 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,301 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T17:39:45,379 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:45,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:45,383 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T17:39:45,389 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:45,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:45,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T17:39:45,394 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:45,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:45,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T17:39:45,401 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:45,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:45,402 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,408 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,410 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,417 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,418 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,424 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T17:39:45,429 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:39:45,436 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:45,437 INFO [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69900616, jitterRate=0.04160034656524658}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T17:39:45,445 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731692385315Initializing all the Stores at 1731692385322 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692385323 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692385324 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692385325 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692385325Cleaning up temporary data from old regions at 1731692385418 (+93 ms)Region opened successfully at 1731692385445 (+27 ms) 2024-11-15T17:39:45,447 INFO [master/4e3b17656000:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T17:39:45,487 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@353f3d52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4e3b17656000/172.17.0.2:0 2024-11-15T17:39:45,520 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T17:39:45,532 INFO [master/4e3b17656000:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T17:39:45,532 INFO [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T17:39:45,535 INFO [master/4e3b17656000:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T17:39:45,536 INFO [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T17:39:45,543 INFO [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T17:39:45,543 INFO [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T17:39:45,578 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T17:39:45,590 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T17:39:45,638 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T17:39:45,640 INFO [master/4e3b17656000:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T17:39:45,642 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T17:39:45,652 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T17:39:45,654 INFO [master/4e3b17656000:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T17:39:45,658 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T17:39:45,668 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T17:39:45,671 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T17:39:45,679 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T17:39:45,698 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T17:39:45,704 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T17:39:45,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T17:39:45,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T17:39:45,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:45,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:45,716 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4e3b17656000,41205,1731692382118, sessionid=0x10140de572c0000, setting cluster-up flag (Was=false) 2024-11-15T17:39:45,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:45,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:45,772 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T17:39:45,776 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4e3b17656000,41205,1731692382118 2024-11-15T17:39:45,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:45,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:45,821 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T17:39:45,824 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4e3b17656000,41205,1731692382118 2024-11-15T17:39:45,832 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T17:39:45,905 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T17:39:45,915 INFO [master/4e3b17656000:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T17:39:45,921 INFO [master/4e3b17656000:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T17:39:45,924 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(746): ClusterId : f848a115-bc41-4153-aa8f-4783b69d52ca 2024-11-15T17:39:45,926 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T17:39:45,927 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4e3b17656000,41205,1731692382118 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T17:39:45,954 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T17:39:45,955 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T17:39:45,955 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4e3b17656000:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:39:45,956 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4e3b17656000:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:39:45,956 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4e3b17656000:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:39:45,956 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4e3b17656000:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:39:45,957 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4e3b17656000:0, corePoolSize=10, maxPoolSize=10 2024-11-15T17:39:45,957 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:45,957 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4e3b17656000:0, corePoolSize=2, maxPoolSize=2 2024-11-15T17:39:45,958 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:45,964 INFO [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731692415963 2024-11-15T17:39:45,965 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T17:39:45,965 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T17:39:45,965 DEBUG [RS:0;4e3b17656000:45187 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47990e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4e3b17656000/172.17.0.2:0 2024-11-15T17:39:45,965 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T17:39:45,966 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T17:39:45,967 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T17:39:45,970 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T17:39:45,970 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T17:39:45,971 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T17:39:45,971 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T17:39:45,972 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:45,971 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:45,972 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T17:39:45,975 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T17:39:45,976 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T17:39:45,976 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T17:39:45,979 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T17:39:45,979 INFO [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T17:39:45,980 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4e3b17656000:45187 2024-11-15T17:39:45,981 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4e3b17656000:0:becomeActiveMaster-HFileCleaner.large.0-1731692385980,5,FailOnTimeoutGroup] 2024-11-15T17:39:45,981 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4e3b17656000:0:becomeActiveMaster-HFileCleaner.small.0-1731692385981,5,FailOnTimeoutGroup] 2024-11-15T17:39:45,981 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:45,982 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T17:39:45,983 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:45,983 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:45,984 INFO [RS:0;4e3b17656000:45187 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T17:39:45,984 INFO [RS:0;4e3b17656000:45187 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T17:39:45,984 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T17:39:45,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741831_1007 (size=1321) 2024-11-15T17:39:45,987 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(2659): reportForDuty to master=4e3b17656000,41205,1731692382118 with port=45187, startcode=1731692383005 2024-11-15T17:39:45,996 DEBUG [RS:0;4e3b17656000:45187 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T17:39:46,028 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T17:39:46,034 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41205 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4e3b17656000,45187,1731692383005 2024-11-15T17:39:46,036 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41205 {}] master.ServerManager(517): Registering regionserver=4e3b17656000,45187,1731692383005 2024-11-15T17:39:46,048 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:39:46,048 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38333 2024-11-15T17:39:46,048 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T17:39:46,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T17:39:46,081 DEBUG [RS:0;4e3b17656000:45187 {}] zookeeper.ZKUtil(111): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4e3b17656000,45187,1731692383005 2024-11-15T17:39:46,081 WARN [RS:0;4e3b17656000:45187 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T17:39:46,082 INFO [RS:0;4e3b17656000:45187 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T17:39:46,082 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/WALs/4e3b17656000,45187,1731692383005 2024-11-15T17:39:46,084 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4e3b17656000,45187,1731692383005] 2024-11-15T17:39:46,109 INFO [RS:0;4e3b17656000:45187 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T17:39:46,124 INFO [RS:0;4e3b17656000:45187 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T17:39:46,129 INFO [RS:0;4e3b17656000:45187 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T17:39:46,129 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,130 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T17:39:46,137 INFO [RS:0;4e3b17656000:45187 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T17:39:46,138 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,138 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,138 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,139 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,139 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,139 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,139 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4e3b17656000:0, corePoolSize=2, maxPoolSize=2 2024-11-15T17:39:46,139 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,139 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,139 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,140 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,140 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,140 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4e3b17656000:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:39:46,140 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4e3b17656000:0, corePoolSize=3, maxPoolSize=3 2024-11-15T17:39:46,140 DEBUG [RS:0;4e3b17656000:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0, corePoolSize=3, maxPoolSize=3 2024-11-15T17:39:46,141 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,142 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,142 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,142 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,142 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,142 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,45187,1731692383005-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T17:39:46,160 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T17:39:46,162 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,45187,1731692383005-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,162 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,162 INFO [RS:0;4e3b17656000:45187 {}] regionserver.Replication(171): 4e3b17656000,45187,1731692383005 started 2024-11-15T17:39:46,178 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:46,179 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1482): Serving as 4e3b17656000,45187,1731692383005, RpcServer on 4e3b17656000/172.17.0.2:45187, sessionid=0x10140de572c0001 2024-11-15T17:39:46,179 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T17:39:46,180 DEBUG [RS:0;4e3b17656000:45187 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4e3b17656000,45187,1731692383005 2024-11-15T17:39:46,180 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4e3b17656000,45187,1731692383005' 2024-11-15T17:39:46,180 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T17:39:46,181 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T17:39:46,182 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T17:39:46,182 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T17:39:46,182 DEBUG [RS:0;4e3b17656000:45187 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4e3b17656000,45187,1731692383005 2024-11-15T17:39:46,182 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4e3b17656000,45187,1731692383005' 2024-11-15T17:39:46,182 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T17:39:46,183 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T17:39:46,183 DEBUG [RS:0;4e3b17656000:45187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T17:39:46,184 INFO [RS:0;4e3b17656000:45187 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T17:39:46,184 INFO [RS:0;4e3b17656000:45187 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T17:39:46,292 INFO [RS:0;4e3b17656000:45187 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T17:39:46,296 INFO [RS:0;4e3b17656000:45187 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4e3b17656000%2C45187%2C1731692383005, suffix=, logDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/WALs/4e3b17656000,45187,1731692383005, archiveDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/oldWALs, maxLogs=32 2024-11-15T17:39:46,313 DEBUG [RS:0;4e3b17656000:45187 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/WALs/4e3b17656000,45187,1731692383005/4e3b17656000%2C45187%2C1731692383005.1731692386300, exclude list is [], retry=0 2024-11-15T17:39:46,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43115,DS-691194fc-f13b-4ebb-87e2-4599068e9e0a,DISK] 2024-11-15T17:39:46,322 INFO [RS:0;4e3b17656000:45187 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/WALs/4e3b17656000,45187,1731692383005/4e3b17656000%2C45187%2C1731692383005.1731692386300 2024-11-15T17:39:46,323 DEBUG [RS:0;4e3b17656000:45187 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42963:42963)] 2024-11-15T17:39:46,389 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T17:39:46,390 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:39:46,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741833_1009 (size=32) 2024-11-15T17:39:46,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:46,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T17:39:46,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T17:39:46,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:46,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:46,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T17:39:46,820 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T17:39:46,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:46,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:46,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T17:39:46,824 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T17:39:46,824 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:46,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:46,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T17:39:46,828 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T17:39:46,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:46,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:46,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T17:39:46,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740 2024-11-15T17:39:46,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740 2024-11-15T17:39:46,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T17:39:46,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T17:39:46,834 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:39:46,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T17:39:46,840 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:46,841 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63246644, jitterRate=-0.057551562786102295}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:39:46,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731692386807Initializing all the Stores at 1731692386812 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692386812Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692386812Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692386812Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692386812Cleaning up temporary data from old regions at 1731692386833 (+21 ms)Region opened successfully at 1731692386843 (+10 ms) 2024-11-15T17:39:46,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T17:39:46,843 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T17:39:46,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T17:39:46,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T17:39:46,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T17:39:46,845 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T17:39:46,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731692386843Disabling compacts and flushes for region at 1731692386843Disabling writes for close at 1731692386843Writing region close event to WAL at 1731692386844 (+1 ms)Closed at 1731692386845 (+1 ms) 2024-11-15T17:39:46,848 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T17:39:46,848 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T17:39:46,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T17:39:46,860 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T17:39:46,863 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T17:39:47,018 DEBUG [4e3b17656000:41205 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T17:39:47,030 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:39:47,036 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4e3b17656000,45187,1731692383005, state=OPENING 2024-11-15T17:39:47,085 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T17:39:47,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:47,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:47,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:47,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:47,102 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T17:39:47,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:39:47,291 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T17:39:47,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60319, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T17:39:47,307 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T17:39:47,307 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T17:39:47,308 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T17:39:47,311 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4e3b17656000%2C45187%2C1731692383005.meta, suffix=.meta, logDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/WALs/4e3b17656000,45187,1731692383005, archiveDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/oldWALs, maxLogs=32 2024-11-15T17:39:47,324 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/WALs/4e3b17656000,45187,1731692383005/4e3b17656000%2C45187%2C1731692383005.meta.1731692387312.meta, exclude list is [], retry=0 2024-11-15T17:39:47,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43115,DS-691194fc-f13b-4ebb-87e2-4599068e9e0a,DISK] 2024-11-15T17:39:47,331 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/WALs/4e3b17656000,45187,1731692383005/4e3b17656000%2C45187%2C1731692383005.meta.1731692387312.meta 2024-11-15T17:39:47,331 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42963:42963)] 2024-11-15T17:39:47,332 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:47,333 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T17:39:47,335 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T17:39:47,339 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T17:39:47,343 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T17:39:47,343 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:47,344 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T17:39:47,344 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T17:39:47,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T17:39:47,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T17:39:47,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:47,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:47,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T17:39:47,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T17:39:47,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:47,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:47,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T17:39:47,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T17:39:47,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:47,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:47,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T17:39:47,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T17:39:47,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:47,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:47,357 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T17:39:47,358 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740 2024-11-15T17:39:47,361 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740 2024-11-15T17:39:47,363 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T17:39:47,363 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T17:39:47,364 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:39:47,367 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T17:39:47,369 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69492552, jitterRate=0.03551971912384033}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:39:47,369 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T17:39:47,370 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731692387344Writing region info on filesystem at 1731692387344Initializing all the Stores at 1731692387346 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692387346Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692387346Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692387346Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692387346Cleaning up temporary data from old regions at 1731692387363 (+17 ms)Running coprocessor post-open hooks at 1731692387369 (+6 ms)Region opened successfully at 1731692387370 (+1 ms) 2024-11-15T17:39:47,376 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731692387281 2024-11-15T17:39:47,386 DEBUG [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T17:39:47,387 INFO [RS_OPEN_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T17:39:47,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:39:47,391 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4e3b17656000,45187,1731692383005, state=OPEN 2024-11-15T17:39:47,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T17:39:47,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T17:39:47,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:47,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:47,436 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:47,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T17:39:47,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4e3b17656000,45187,1731692383005 in 330 msec 2024-11-15T17:39:47,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T17:39:47,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 594 msec 2024-11-15T17:39:47,453 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T17:39:47,453 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T17:39:47,471 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:47,472 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:47,491 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:47,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36275, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:47,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6510 sec 2024-11-15T17:39:47,515 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731692387514, completionTime=-1 2024-11-15T17:39:47,517 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T17:39:47,517 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T17:39:47,541 INFO [master/4e3b17656000:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T17:39:47,541 INFO [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731692447541 2024-11-15T17:39:47,541 INFO [master/4e3b17656000:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731692507541 2024-11-15T17:39:47,541 INFO [master/4e3b17656000:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-15T17:39:47,543 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,41205,1731692382118-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:47,543 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,41205,1731692382118-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:47,543 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,41205,1731692382118-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:47,545 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4e3b17656000:41205, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:47,545 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:47,546 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:47,551 DEBUG [master/4e3b17656000:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T17:39:47,570 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.375sec 2024-11-15T17:39:47,571 INFO [master/4e3b17656000:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T17:39:47,572 INFO [master/4e3b17656000:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T17:39:47,573 INFO [master/4e3b17656000:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T17:39:47,573 INFO [master/4e3b17656000:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T17:39:47,573 INFO [master/4e3b17656000:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T17:39:47,574 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,41205,1731692382118-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T17:39:47,575 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,41205,1731692382118-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T17:39:47,582 DEBUG [master/4e3b17656000:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T17:39:47,583 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T17:39:47,584 INFO [master/4e3b17656000:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4e3b17656000,41205,1731692382118-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:47,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:47,641 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T17:39:47,641 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T17:39:47,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:47,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:47,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:47,665 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:47,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:47,669 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:47,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:47,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4e3b17656000,41205,1731692382118 2024-11-15T17:39:47,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=295, ProcessCount=11, AvailableMemoryMB=10174 2024-11-15T17:39:47,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:39:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4e3b17656000,41205,1731692382118 2024-11-15T17:39:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-15T17:39:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:39:47,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:39:47,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:39:47,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T17:39:47,791 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:47,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T17:39:47,794 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:47,797 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:47,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741835_1011 (size=959) 2024-11-15T17:39:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:48,215 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:39:48,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741836_1012 (size=53) 2024-11-15T17:39:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:48,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:48,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ada8f074c978a899b609b6215cb2d474, disabling compactions & flushes 2024-11-15T17:39:48,633 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:48,633 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:48,633 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. after waiting 0 ms 2024-11-15T17:39:48,633 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:48,634 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:48,634 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ada8f074c978a899b609b6215cb2d474: Waiting for close lock at 1731692388632Disabling compacts and flushes for region at 1731692388632Disabling writes for close at 1731692388633 (+1 ms)Writing region close event to WAL at 1731692388633Closed at 1731692388633 2024-11-15T17:39:48,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:48,644 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731692388638"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692388638"}]},"ts":"1731692388638"} 2024-11-15T17:39:48,649 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:48,650 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:48,653 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692388651"}]},"ts":"1731692388651"} 2024-11-15T17:39:48,657 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T17:39:48,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, ASSIGN}] 2024-11-15T17:39:48,662 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, ASSIGN 2024-11-15T17:39:48,664 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, ASSIGN; state=OFFLINE, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=false 2024-11-15T17:39:48,817 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ada8f074c978a899b609b6215cb2d474, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:39:48,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, ASSIGN because future has completed 2024-11-15T17:39:48,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:39:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:48,993 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:48,994 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:48,994 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:48,995 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:48,995 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:48,995 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:48,997 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,000 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ada8f074c978a899b609b6215cb2d474 columnFamilyName A 2024-11-15T17:39:49,000 DEBUG [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:49,001 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] regionserver.HStore(327): Store=ada8f074c978a899b609b6215cb2d474/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:49,001 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,003 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ada8f074c978a899b609b6215cb2d474 columnFamilyName B 2024-11-15T17:39:49,003 DEBUG [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:49,004 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] regionserver.HStore(327): Store=ada8f074c978a899b609b6215cb2d474/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:49,004 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,006 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ada8f074c978a899b609b6215cb2d474 columnFamilyName C 2024-11-15T17:39:49,006 DEBUG [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:49,007 INFO [StoreOpener-ada8f074c978a899b609b6215cb2d474-1 {}] regionserver.HStore(327): Store=ada8f074c978a899b609b6215cb2d474/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:49,007 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,009 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,009 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,011 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,011 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,012 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:39:49,015 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,018 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:49,019 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ada8f074c978a899b609b6215cb2d474; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63411673, jitterRate=-0.05509243905544281}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:39:49,019 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:49,020 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ada8f074c978a899b609b6215cb2d474: Running coprocessor pre-open hook at 1731692388995Writing region info on filesystem at 1731692388995Initializing all the Stores at 1731692388997 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692388997Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692388997Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692388997Cleaning up temporary data from old regions at 1731692389011 (+14 ms)Running coprocessor post-open hooks at 1731692389019 (+8 ms)Region opened successfully at 1731692389020 (+1 ms) 2024-11-15T17:39:49,023 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., pid=6, masterSystemTime=1731692388983 2024-11-15T17:39:49,026 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:49,026 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:49,028 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ada8f074c978a899b609b6215cb2d474, regionState=OPEN, openSeqNum=2, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:39:49,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:39:49,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T17:39:49,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 in 206 msec 2024-11-15T17:39:49,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T17:39:49,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, ASSIGN in 379 msec 2024-11-15T17:39:49,046 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:49,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692389046"}]},"ts":"1731692389046"} 2024-11-15T17:39:49,049 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T17:39:49,052 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:49,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2750 sec 2024-11-15T17:39:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:49,957 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:39:49,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:49,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:49,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:49,965 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec19887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:49,968 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:49,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621eb0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:49,971 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:49,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:49,979 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:39:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4e3b17656000,41205,1731692382118 2024-11-15T17:39:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@254705f9 2024-11-15T17:39:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:39:49,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:39:49,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:49,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:49,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:49,992 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c72f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:49,995 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:49,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745237a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:49,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:49,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:49,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:49,999 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,003 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:50,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:50,008 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,012 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:50,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:50,016 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,019 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2815dd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:50,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:50,022 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1214a151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,024 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:50,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:50,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:50,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:50,034 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555d2363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,036 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a886a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:50,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:50,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e23982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,042 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663b0bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:39:50,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:50,045 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:39:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:39:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe80a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:39:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:50,048 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213c7631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:50,054 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,054 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,055 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,055 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,055 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,056 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,063 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,075 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,085 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:50,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:39:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:50,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:50,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:39:50,101 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:39:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T17:39:50,112 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:39:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:50,116 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:39:50,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:39:50,159 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:39:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:50,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:39:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:50,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/df99f74e17014ebeb6a46554fe51d0a2 is 50, key is test_row_0/A:col10/1731692390151/Put/seqid=0 2024-11-15T17:39:50,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:50,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741837_1013 (size=12001) 2024-11-15T17:39:50,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/df99f74e17014ebeb6a46554fe51d0a2 2024-11-15T17:39:50,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692450357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692450356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692450369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692450378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692450381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:50,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:50,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/e97f6f0ec23e4048af8eff916de980ec is 50, key is test_row_0/B:col10/1731692390151/Put/seqid=0 2024-11-15T17:39:50,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741838_1014 (size=12001) 2024-11-15T17:39:50,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692450537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692450538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692450539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692450539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692450538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:50,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:50,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:50,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692450757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692450757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692450758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692450758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692450758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:50,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:50,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:50,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/e97f6f0ec23e4048af8eff916de980ec 2024-11-15T17:39:50,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:50,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:50,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:50,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:50,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/1ea68ad278514677969cbe5cf8974e10 is 50, key is test_row_0/C:col10/1731692390151/Put/seqid=0 2024-11-15T17:39:51,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741839_1015 (size=12001) 2024-11-15T17:39:51,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692451068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692451068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692451068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692451066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692451076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:51,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:51,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:51,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/1ea68ad278514677969cbe5cf8974e10 2024-11-15T17:39:51,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/df99f74e17014ebeb6a46554fe51d0a2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/df99f74e17014ebeb6a46554fe51d0a2 2024-11-15T17:39:51,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/df99f74e17014ebeb6a46554fe51d0a2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T17:39:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/e97f6f0ec23e4048af8eff916de980ec as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e97f6f0ec23e4048af8eff916de980ec 2024-11-15T17:39:51,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e97f6f0ec23e4048af8eff916de980ec, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T17:39:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/1ea68ad278514677969cbe5cf8974e10 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1ea68ad278514677969cbe5cf8974e10 2024-11-15T17:39:51,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1ea68ad278514677969cbe5cf8974e10, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T17:39:51,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ada8f074c978a899b609b6215cb2d474 in 1340ms, sequenceid=13, compaction requested=false 2024-11-15T17:39:51,509 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T17:39:51,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:51,583 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:39:51,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:39:51,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/7706b57d2f4b42cfaea688a7df571bc3 is 50, key is test_row_0/A:col10/1731692390354/Put/seqid=0 2024-11-15T17:39:51,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741840_1016 (size=14341) 2024-11-15T17:39:51,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/7706b57d2f4b42cfaea688a7df571bc3 2024-11-15T17:39:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692451605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692451606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692451615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692451616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692451620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/fa459ba46d9844dcbe0567ce3e6a40b4 is 50, key is test_row_0/B:col10/1731692390354/Put/seqid=0 2024-11-15T17:39:51,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741841_1017 (size=12001) 2024-11-15T17:39:51,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692451727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692451727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692451746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692451747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692451747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:51,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692451946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692451946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692451966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692451968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692451969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/fa459ba46d9844dcbe0567ce3e6a40b4 2024-11-15T17:39:52,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T17:39:52,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/aaa0318194cd42b998f0ddd7838f7491 is 50, key is test_row_0/C:col10/1731692390354/Put/seqid=0 2024-11-15T17:39:52,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741842_1018 (size=12001) 2024-11-15T17:39:52,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692452257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692452257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692452277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692452277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692452286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:52,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:52,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:52,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/aaa0318194cd42b998f0ddd7838f7491 2024-11-15T17:39:52,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/7706b57d2f4b42cfaea688a7df571bc3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/7706b57d2f4b42cfaea688a7df571bc3 2024-11-15T17:39:52,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/7706b57d2f4b42cfaea688a7df571bc3, entries=200, sequenceid=38, filesize=14.0 K 2024-11-15T17:39:52,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/fa459ba46d9844dcbe0567ce3e6a40b4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/fa459ba46d9844dcbe0567ce3e6a40b4 2024-11-15T17:39:52,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/fa459ba46d9844dcbe0567ce3e6a40b4, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T17:39:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/aaa0318194cd42b998f0ddd7838f7491 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/aaa0318194cd42b998f0ddd7838f7491 2024-11-15T17:39:52,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/aaa0318194cd42b998f0ddd7838f7491, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T17:39:52,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ada8f074c978a899b609b6215cb2d474 in 1087ms, sequenceid=38, compaction requested=false 2024-11-15T17:39:52,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:52,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T17:39:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:52,697 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:39:52,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/1b1934072f35478cbc53c7bdcd79a081 is 50, key is test_row_1/A:col10/1731692391609/Put/seqid=0 2024-11-15T17:39:52,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741843_1019 (size=9657) 2024-11-15T17:39:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:52,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:52,857 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:39:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692452883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692452879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692452883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692452902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692452904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:52,979 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T17:39:52,979 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:52,982 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T17:39:52,982 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T17:39:52,983 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T17:39:52,983 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:52,985 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T17:39:52,985 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692453017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692453016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692453017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692453018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692453018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,131 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/1b1934072f35478cbc53c7bdcd79a081 2024-11-15T17:39:53,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/7e9a825e4f5549b88026a13010b21a9c is 50, key is test_row_1/B:col10/1731692391609/Put/seqid=0 2024-11-15T17:39:53,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741844_1020 (size=9657) 2024-11-15T17:39:53,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692453236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692453238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692453238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,256 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/7e9a825e4f5549b88026a13010b21a9c 2024-11-15T17:39:53,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692453246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692453247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/b7561729deaa464b9daa684b6e2b2cc0 is 50, key is test_row_1/C:col10/1731692391609/Put/seqid=0 2024-11-15T17:39:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741845_1021 (size=9657) 2024-11-15T17:39:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692453547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692453547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692453567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692453577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692453577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:53,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:53,747 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/b7561729deaa464b9daa684b6e2b2cc0 2024-11-15T17:39:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/1b1934072f35478cbc53c7bdcd79a081 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b1934072f35478cbc53c7bdcd79a081 2024-11-15T17:39:53,779 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b1934072f35478cbc53c7bdcd79a081, entries=100, sequenceid=49, filesize=9.4 K 2024-11-15T17:39:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/7e9a825e4f5549b88026a13010b21a9c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7e9a825e4f5549b88026a13010b21a9c 2024-11-15T17:39:53,808 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7e9a825e4f5549b88026a13010b21a9c, entries=100, sequenceid=49, filesize=9.4 K 2024-11-15T17:39:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/b7561729deaa464b9daa684b6e2b2cc0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/b7561729deaa464b9daa684b6e2b2cc0 2024-11-15T17:39:53,827 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/b7561729deaa464b9daa684b6e2b2cc0, entries=100, sequenceid=49, filesize=9.4 K 2024-11-15T17:39:53,833 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ada8f074c978a899b609b6215cb2d474 in 1135ms, sequenceid=49, compaction requested=true 2024-11-15T17:39:53,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:53,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:53,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T17:39:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T17:39:53,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T17:39:53,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7190 sec 2024-11-15T17:39:53,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.7420 sec 2024-11-15T17:39:54,075 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:39:54,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T17:39:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:54,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/898bd54bb88848959fd2f489403b7fc6 is 50, key is test_row_0/A:col10/1731692394073/Put/seqid=0 2024-11-15T17:39:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692454120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692454120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692454124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692454126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692454126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741846_1022 (size=14341) 2024-11-15T17:39:54,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/898bd54bb88848959fd2f489403b7fc6 2024-11-15T17:39:54,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/1f9d00ec302e4b15b0158ea4d928fdbc is 50, key is test_row_0/B:col10/1731692394073/Put/seqid=0 2024-11-15T17:39:54,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692454245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692454246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741847_1023 (size=12001) 2024-11-15T17:39:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692454246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692454246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692454246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:54,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:39:54,289 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:39:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T17:39:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T17:39:54,294 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:39:54,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:39:54,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:39:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T17:39:54,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692454456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692454456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T17:39:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692454468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692454468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692454468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T17:39:54,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T17:39:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/1f9d00ec302e4b15b0158ea4d928fdbc 2024-11-15T17:39:54,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/da78062f6e9342ada1dd6a7e5e117998 is 50, key is test_row_0/C:col10/1731692394073/Put/seqid=0 2024-11-15T17:39:54,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741848_1024 (size=12001) 2024-11-15T17:39:54,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/da78062f6e9342ada1dd6a7e5e117998 2024-11-15T17:39:54,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/898bd54bb88848959fd2f489403b7fc6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/898bd54bb88848959fd2f489403b7fc6 2024-11-15T17:39:54,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692454767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692454772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T17:39:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/898bd54bb88848959fd2f489403b7fc6, entries=200, sequenceid=76, filesize=14.0 K 2024-11-15T17:39:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:54,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/1f9d00ec302e4b15b0158ea4d928fdbc as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1f9d00ec302e4b15b0158ea4d928fdbc 2024-11-15T17:39:54,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1f9d00ec302e4b15b0158ea4d928fdbc, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T17:39:54,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/da78062f6e9342ada1dd6a7e5e117998 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/da78062f6e9342ada1dd6a7e5e117998 2024-11-15T17:39:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692454776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692454776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692454776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:54,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:54,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/da78062f6e9342ada1dd6a7e5e117998, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T17:39:54,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for ada8f074c978a899b609b6215cb2d474 in 736ms, sequenceid=76, compaction requested=true 2024-11-15T17:39:54,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:54,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:39:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:39:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:39:54,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:39:54,835 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:39:54,837 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:39:54,842 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:39:54,844 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:39:54,845 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,845 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e97f6f0ec23e4048af8eff916de980ec, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/fa459ba46d9844dcbe0567ce3e6a40b4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7e9a825e4f5549b88026a13010b21a9c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1f9d00ec302e4b15b0158ea4d928fdbc] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=44.6 K 2024-11-15T17:39:54,848 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting e97f6f0ec23e4048af8eff916de980ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731692390134 2024-11-15T17:39:54,853 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:39:54,853 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fa459ba46d9844dcbe0567ce3e6a40b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731692390354 2024-11-15T17:39:54,853 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:39:54,853 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,853 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/df99f74e17014ebeb6a46554fe51d0a2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/7706b57d2f4b42cfaea688a7df571bc3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b1934072f35478cbc53c7bdcd79a081, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/898bd54bb88848959fd2f489403b7fc6] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=49.2 K 2024-11-15T17:39:54,854 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9a825e4f5549b88026a13010b21a9c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731692391609 2024-11-15T17:39:54,854 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting df99f74e17014ebeb6a46554fe51d0a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731692390134 2024-11-15T17:39:54,856 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7706b57d2f4b42cfaea688a7df571bc3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731692390216 2024-11-15T17:39:54,856 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f9d00ec302e4b15b0158ea4d928fdbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731692392875 2024-11-15T17:39:54,858 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b1934072f35478cbc53c7bdcd79a081, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731692391609 2024-11-15T17:39:54,860 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 898bd54bb88848959fd2f489403b7fc6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731692392875 2024-11-15T17:39:54,919 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:54,919 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#12 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:54,920 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/2ea167a40821480c8f690e2cc26ff9ce is 50, key is test_row_0/B:col10/1731692394073/Put/seqid=0 2024-11-15T17:39:54,925 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/e6ebe6b329014218ac0860ec472fb1cb is 50, key is test_row_0/A:col10/1731692394073/Put/seqid=0 2024-11-15T17:39:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T17:39:54,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T17:39:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:54,931 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T17:39:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/514337ae287d4302b56edb30438c4eec is 50, key is test_row_0/A:col10/1731692394123/Put/seqid=0 2024-11-15T17:39:55,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741849_1025 (size=12139) 2024-11-15T17:39:55,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741850_1026 (size=12139) 2024-11-15T17:39:55,046 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/2ea167a40821480c8f690e2cc26ff9ce as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2ea167a40821480c8f690e2cc26ff9ce 2024-11-15T17:39:55,047 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/e6ebe6b329014218ac0860ec472fb1cb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e6ebe6b329014218ac0860ec472fb1cb 2024-11-15T17:39:55,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741851_1027 (size=12001) 2024-11-15T17:39:55,080 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into 2ea167a40821480c8f690e2cc26ff9ce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:55,080 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:55,083 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into e6ebe6b329014218ac0860ec472fb1cb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:55,083 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:55,084 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=12, startTime=1731692394813; duration=0sec 2024-11-15T17:39:55,085 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:39:55,085 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:39:55,086 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:39:55,086 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=12, startTime=1731692394834; duration=0sec 2024-11-15T17:39:55,087 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:55,087 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:39:55,094 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:39:55,094 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:39:55,094 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:55,094 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1ea68ad278514677969cbe5cf8974e10, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/aaa0318194cd42b998f0ddd7838f7491, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/b7561729deaa464b9daa684b6e2b2cc0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/da78062f6e9342ada1dd6a7e5e117998] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=44.6 K 2024-11-15T17:39:55,096 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea68ad278514677969cbe5cf8974e10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731692390134 2024-11-15T17:39:55,097 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaa0318194cd42b998f0ddd7838f7491, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731692390354 2024-11-15T17:39:55,098 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7561729deaa464b9daa684b6e2b2cc0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731692391609 2024-11-15T17:39:55,099 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting da78062f6e9342ada1dd6a7e5e117998, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731692392875 2024-11-15T17:39:55,147 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#15 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:55,148 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/8013f06723384f5286f07376e862422f is 50, key is test_row_0/C:col10/1731692394073/Put/seqid=0 2024-11-15T17:39:55,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741852_1028 (size=12139) 2024-11-15T17:39:55,212 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/8013f06723384f5286f07376e862422f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8013f06723384f5286f07376e862422f 2024-11-15T17:39:55,228 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into 8013f06723384f5286f07376e862422f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:55,228 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:55,229 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=12, startTime=1731692394834; duration=0sec 2024-11-15T17:39:55,229 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:55,229 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:39:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:55,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692455399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692455403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692455405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692455408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692455412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T17:39:55,459 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/514337ae287d4302b56edb30438c4eec 2024-11-15T17:39:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/aaa7b29e159e4527a5708c41fc0248b4 is 50, key is test_row_0/B:col10/1731692394123/Put/seqid=0 2024-11-15T17:39:55,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741853_1029 (size=12001) 2024-11-15T17:39:55,504 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/aaa7b29e159e4527a5708c41fc0248b4 2024-11-15T17:39:55,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692455517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692455526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692455526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692455526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692455527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/cd76dd13f8e44675b4529a1d5fb6fb65 is 50, key is test_row_0/C:col10/1731692394123/Put/seqid=0 2024-11-15T17:39:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741854_1030 (size=12001) 2024-11-15T17:39:55,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692455736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692455736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692455736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692455746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692455746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:55,961 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/cd76dd13f8e44675b4529a1d5fb6fb65 2024-11-15T17:39:55,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/514337ae287d4302b56edb30438c4eec as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/514337ae287d4302b56edb30438c4eec 2024-11-15T17:39:55,987 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/514337ae287d4302b56edb30438c4eec, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T17:39:55,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/aaa7b29e159e4527a5708c41fc0248b4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/aaa7b29e159e4527a5708c41fc0248b4 2024-11-15T17:39:56,001 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/aaa7b29e159e4527a5708c41fc0248b4, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T17:39:56,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/cd76dd13f8e44675b4529a1d5fb6fb65 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/cd76dd13f8e44675b4529a1d5fb6fb65 2024-11-15T17:39:56,016 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/cd76dd13f8e44675b4529a1d5fb6fb65, entries=150, sequenceid=85, filesize=11.7 K 2024-11-15T17:39:56,018 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for ada8f074c978a899b609b6215cb2d474 in 1087ms, sequenceid=85, compaction requested=false 2024-11-15T17:39:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T17:39:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T17:39:56,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T17:39:56,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7230 sec 2024-11-15T17:39:56,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7390 sec 2024-11-15T17:39:56,050 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:39:56,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T17:39:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:56,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/513e722ca03c46e8a1612271ad605df3 is 50, key is test_row_0/A:col10/1731692395402/Put/seqid=0 2024-11-15T17:39:56,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692456058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692456060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692456073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692456076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692456076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741855_1031 (size=14341) 2024-11-15T17:39:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692456186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692456186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692456197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692456197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692456197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692456396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692456400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692456407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692456408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692456416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T17:39:56,446 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:39:56,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:39:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T17:39:56,452 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:39:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T17:39:56,453 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:39:56,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:39:56,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/513e722ca03c46e8a1612271ad605df3 2024-11-15T17:39:56,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/435d6374d80c4f929d86a02d651a9279 is 50, key is test_row_0/B:col10/1731692395402/Put/seqid=0 2024-11-15T17:39:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T17:39:56,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741856_1032 (size=12001) 2024-11-15T17:39:56,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T17:39:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:56,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692456706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692456716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692456716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692456717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692456726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:56,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T17:39:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T17:39:56,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T17:39:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:56,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:56,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/435d6374d80c4f929d86a02d651a9279 2024-11-15T17:39:57,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/4232c04dfc404c2cb13cfa4d1477712a is 50, key is test_row_0/C:col10/1731692395402/Put/seqid=0 2024-11-15T17:39:57,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741857_1033 (size=12001) 2024-11-15T17:39:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T17:39:57,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T17:39:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692457216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:57,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692457229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:57,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:57,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692457229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692457229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:57,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T17:39:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692457236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:57,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T17:39:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:57,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:57,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/4232c04dfc404c2cb13cfa4d1477712a 2024-11-15T17:39:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/513e722ca03c46e8a1612271ad605df3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/513e722ca03c46e8a1612271ad605df3 2024-11-15T17:39:57,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/513e722ca03c46e8a1612271ad605df3, entries=200, sequenceid=116, filesize=14.0 K 2024-11-15T17:39:57,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/435d6374d80c4f929d86a02d651a9279 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/435d6374d80c4f929d86a02d651a9279 2024-11-15T17:39:57,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/435d6374d80c4f929d86a02d651a9279, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T17:39:57,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/4232c04dfc404c2cb13cfa4d1477712a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/4232c04dfc404c2cb13cfa4d1477712a 2024-11-15T17:39:57,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/4232c04dfc404c2cb13cfa4d1477712a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T17:39:57,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for ada8f074c978a899b609b6215cb2d474 in 1483ms, sequenceid=116, compaction requested=true 2024-11-15T17:39:57,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:57,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:39:57,535 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:39:57,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:57,535 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:39:57,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:39:57,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:39:57,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:39:57,537 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:39:57,537 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:39:57,537 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:39:57,537 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:39:57,537 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,538 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e6ebe6b329014218ac0860ec472fb1cb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/514337ae287d4302b56edb30438c4eec, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/513e722ca03c46e8a1612271ad605df3] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=37.6 K 2024-11-15T17:39:57,538 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,538 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2ea167a40821480c8f690e2cc26ff9ce, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/aaa7b29e159e4527a5708c41fc0248b4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/435d6374d80c4f929d86a02d651a9279] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=35.3 K 2024-11-15T17:39:57,538 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6ebe6b329014218ac0860ec472fb1cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731692392875 2024-11-15T17:39:57,539 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 514337ae287d4302b56edb30438c4eec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731692394116 2024-11-15T17:39:57,539 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea167a40821480c8f690e2cc26ff9ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731692392875 2024-11-15T17:39:57,540 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 513e722ca03c46e8a1612271ad605df3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692395378 2024-11-15T17:39:57,540 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting aaa7b29e159e4527a5708c41fc0248b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731692394116 2024-11-15T17:39:57,540 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 435d6374d80c4f929d86a02d651a9279, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692395378 2024-11-15T17:39:57,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T17:39:57,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,547 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T17:39:57,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/39b81b920e1a4c47a4903fd66ee49c65 is 50, key is test_row_0/A:col10/1731692396060/Put/seqid=0 2024-11-15T17:39:57,571 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:57,572 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/a8ff44e4bb184d538ec68e8b2e9a4571 is 50, key is test_row_0/A:col10/1731692395402/Put/seqid=0 2024-11-15T17:39:57,572 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:57,573 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/e4d87e9520234731946f43c5776ecd65 is 50, key is test_row_0/B:col10/1731692395402/Put/seqid=0 2024-11-15T17:39:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T17:39:57,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741858_1034 (size=9657) 2024-11-15T17:39:57,599 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/39b81b920e1a4c47a4903fd66ee49c65 2024-11-15T17:39:57,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741859_1035 (size=12241) 2024-11-15T17:39:57,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741860_1036 (size=12241) 2024-11-15T17:39:57,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/5d645bb88ba84e5c9a9fceae5b42654d is 50, key is test_row_0/B:col10/1731692396060/Put/seqid=0 2024-11-15T17:39:57,657 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/a8ff44e4bb184d538ec68e8b2e9a4571 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/a8ff44e4bb184d538ec68e8b2e9a4571 2024-11-15T17:39:57,657 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/e4d87e9520234731946f43c5776ecd65 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e4d87e9520234731946f43c5776ecd65 2024-11-15T17:39:57,677 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into e4d87e9520234731946f43c5776ecd65(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:57,677 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:57,677 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=13, startTime=1731692397535; duration=0sec 2024-11-15T17:39:57,677 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:39:57,677 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:39:57,678 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:39:57,680 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into a8ff44e4bb184d538ec68e8b2e9a4571(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:57,680 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:57,680 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=13, startTime=1731692397535; duration=0sec 2024-11-15T17:39:57,681 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:57,681 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:39:57,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741861_1037 (size=9657) 2024-11-15T17:39:57,682 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:39:57,683 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:39:57,683 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/5d645bb88ba84e5c9a9fceae5b42654d 2024-11-15T17:39:57,683 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:57,683 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8013f06723384f5286f07376e862422f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/cd76dd13f8e44675b4529a1d5fb6fb65, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/4232c04dfc404c2cb13cfa4d1477712a] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=35.3 K 2024-11-15T17:39:57,684 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8013f06723384f5286f07376e862422f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731692392875 2024-11-15T17:39:57,684 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting cd76dd13f8e44675b4529a1d5fb6fb65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731692394116 2024-11-15T17:39:57,685 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4232c04dfc404c2cb13cfa4d1477712a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692395378 2024-11-15T17:39:57,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/85a72e6e40644fc799b3a0e03464ca52 is 50, key is test_row_0/C:col10/1731692396060/Put/seqid=0 2024-11-15T17:39:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741862_1038 (size=9657) 2024-11-15T17:39:57,714 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#26 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:57,714 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/2357564ca0a64af3beeb47f207b202fb is 50, key is test_row_0/C:col10/1731692395402/Put/seqid=0 2024-11-15T17:39:57,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741863_1039 (size=12241) 2024-11-15T17:39:58,113 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/85a72e6e40644fc799b3a0e03464ca52 2024-11-15T17:39:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/39b81b920e1a4c47a4903fd66ee49c65 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/39b81b920e1a4c47a4903fd66ee49c65 2024-11-15T17:39:58,144 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/39b81b920e1a4c47a4903fd66ee49c65, entries=100, sequenceid=124, filesize=9.4 K 2024-11-15T17:39:58,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/5d645bb88ba84e5c9a9fceae5b42654d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/5d645bb88ba84e5c9a9fceae5b42654d 2024-11-15T17:39:58,150 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/2357564ca0a64af3beeb47f207b202fb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/2357564ca0a64af3beeb47f207b202fb 2024-11-15T17:39:58,163 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/5d645bb88ba84e5c9a9fceae5b42654d, entries=100, sequenceid=124, filesize=9.4 K 2024-11-15T17:39:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/85a72e6e40644fc799b3a0e03464ca52 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/85a72e6e40644fc799b3a0e03464ca52 2024-11-15T17:39:58,167 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into 2357564ca0a64af3beeb47f207b202fb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:58,167 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:58,168 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=13, startTime=1731692397535; duration=0sec 2024-11-15T17:39:58,168 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:58,168 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:39:58,178 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/85a72e6e40644fc799b3a0e03464ca52, entries=100, sequenceid=124, filesize=9.4 K 2024-11-15T17:39:58,181 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ada8f074c978a899b609b6215cb2d474 in 634ms, sequenceid=124, compaction requested=false 2024-11-15T17:39:58,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:58,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:58,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T17:39:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T17:39:58,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T17:39:58,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7320 sec 2024-11-15T17:39:58,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7450 sec 2024-11-15T17:39:58,250 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:39:58,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:39:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:58,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/2a935101bccb4ebb8c6b8ee103c3dd99 is 50, key is test_row_0/A:col10/1731692398250/Put/seqid=0 2024-11-15T17:39:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692458302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692458298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692458306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692458311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692458311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741864_1040 (size=9757) 2024-11-15T17:39:58,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/2a935101bccb4ebb8c6b8ee103c3dd99 2024-11-15T17:39:58,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/04b2199c6ddd49f8b9273abd4d7fb847 is 50, key is test_row_0/B:col10/1731692398250/Put/seqid=0 2024-11-15T17:39:58,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741865_1041 (size=9757) 2024-11-15T17:39:58,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/04b2199c6ddd49f8b9273abd4d7fb847 2024-11-15T17:39:58,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/96c63591d921445a9dd9f02b6aa99037 is 50, key is test_row_0/C:col10/1731692398250/Put/seqid=0 2024-11-15T17:39:58,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692458416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692458426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692458427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692458426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692458427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741866_1042 (size=9757) 2024-11-15T17:39:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/96c63591d921445a9dd9f02b6aa99037 2024-11-15T17:39:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/2a935101bccb4ebb8c6b8ee103c3dd99 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/2a935101bccb4ebb8c6b8ee103c3dd99 2024-11-15T17:39:58,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/2a935101bccb4ebb8c6b8ee103c3dd99, entries=100, sequenceid=139, filesize=9.5 K 2024-11-15T17:39:58,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/04b2199c6ddd49f8b9273abd4d7fb847 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/04b2199c6ddd49f8b9273abd4d7fb847 2024-11-15T17:39:58,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/04b2199c6ddd49f8b9273abd4d7fb847, entries=100, sequenceid=139, filesize=9.5 K 2024-11-15T17:39:58,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/96c63591d921445a9dd9f02b6aa99037 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/96c63591d921445a9dd9f02b6aa99037 2024-11-15T17:39:58,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/96c63591d921445a9dd9f02b6aa99037, entries=100, sequenceid=139, filesize=9.5 K 2024-11-15T17:39:58,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ada8f074c978a899b609b6215cb2d474 in 238ms, sequenceid=139, compaction requested=true 2024-11-15T17:39:58,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:58,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:39:58,488 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:39:58,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:58,488 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:39:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:39:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:58,491 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:39:58,491 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:39:58,491 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:58,491 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/a8ff44e4bb184d538ec68e8b2e9a4571, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/39b81b920e1a4c47a4903fd66ee49c65, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/2a935101bccb4ebb8c6b8ee103c3dd99] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=30.9 K 2024-11-15T17:39:58,492 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ff44e4bb184d538ec68e8b2e9a4571, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692395378 2024-11-15T17:39:58,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:39:58,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:39:58,493 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b81b920e1a4c47a4903fd66ee49c65, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731692396060 2024-11-15T17:39:58,493 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:39:58,493 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:39:58,493 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:58,494 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e4d87e9520234731946f43c5776ecd65, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/5d645bb88ba84e5c9a9fceae5b42654d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/04b2199c6ddd49f8b9273abd4d7fb847] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=30.9 K 2024-11-15T17:39:58,495 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting e4d87e9520234731946f43c5776ecd65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692395378 2024-11-15T17:39:58,495 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a935101bccb4ebb8c6b8ee103c3dd99, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731692398248 2024-11-15T17:39:58,495 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d645bb88ba84e5c9a9fceae5b42654d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731692396060 2024-11-15T17:39:58,496 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b2199c6ddd49f8b9273abd4d7fb847, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731692398248 2024-11-15T17:39:58,516 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:58,517 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/c83e8d1e8b3a4a2c9690eb1c383b418c is 50, key is test_row_0/B:col10/1731692398250/Put/seqid=0 2024-11-15T17:39:58,533 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:58,534 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/5266f13997ee43cfa5973b11a1704d04 is 50, key is test_row_0/A:col10/1731692398250/Put/seqid=0 2024-11-15T17:39:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741867_1043 (size=12443) 2024-11-15T17:39:58,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741868_1044 (size=12443) 2024-11-15T17:39:58,581 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/5266f13997ee43cfa5973b11a1704d04 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5266f13997ee43cfa5973b11a1704d04 2024-11-15T17:39:58,596 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into 5266f13997ee43cfa5973b11a1704d04(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T17:39:58,596 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:58,596 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=13, startTime=1731692398488; duration=0sec 2024-11-15T17:39:58,596 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:39:58,596 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:39:58,596 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:39:58,596 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:39:58,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:39:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T17:39:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:58,600 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:39:58,601 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:39:58,601 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:39:58,601 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:58,601 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/2357564ca0a64af3beeb47f207b202fb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/85a72e6e40644fc799b3a0e03464ca52, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/96c63591d921445a9dd9f02b6aa99037] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=30.9 K 2024-11-15T17:39:58,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:39:58,602 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2357564ca0a64af3beeb47f207b202fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692395378 2024-11-15T17:39:58,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:39:58,603 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a72e6e40644fc799b3a0e03464ca52, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731692396060 2024-11-15T17:39:58,603 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96c63591d921445a9dd9f02b6aa99037, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731692398248 2024-11-15T17:39:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:58,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:39:58,628 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#32 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:39:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:39:58,633 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/997af595ee794c1098deeba3e732551b is 50, key is test_row_0/C:col10/1731692398250/Put/seqid=0 2024-11-15T17:39:58,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/d152b0f7343048a9905836bb6c16a0de is 50, key is test_row_0/A:col10/1731692398302/Put/seqid=0 2024-11-15T17:39:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692458649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692458653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692458654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741869_1045 (size=12443) 2024-11-15T17:39:58,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692458655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692458657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741870_1046 (size=14541) 2024-11-15T17:39:58,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/d152b0f7343048a9905836bb6c16a0de 2024-11-15T17:39:58,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/37288ac2e5bc40689dc40f4540eac24d is 50, key is test_row_0/B:col10/1731692398302/Put/seqid=0 2024-11-15T17:39:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:58,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741871_1047 (size=12151) 2024-11-15T17:39:58,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/37288ac2e5bc40689dc40f4540eac24d 2024-11-15T17:39:58,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/02c6cb35b57d471cbb85a99f35f7cc2e is 50, key is test_row_0/C:col10/1731692398302/Put/seqid=0 2024-11-15T17:39:58,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741872_1048 (size=12151) 2024-11-15T17:39:58,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/02c6cb35b57d471cbb85a99f35f7cc2e 2024-11-15T17:39:58,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T17:39:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:58,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:58,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:58,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:58,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/d152b0f7343048a9905836bb6c16a0de as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d152b0f7343048a9905836bb6c16a0de 2024-11-15T17:39:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692458766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692458766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692458775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692458775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692458776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:58,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d152b0f7343048a9905836bb6c16a0de, entries=200, sequenceid=165, filesize=14.2 K 2024-11-15T17:39:58,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/37288ac2e5bc40689dc40f4540eac24d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/37288ac2e5bc40689dc40f4540eac24d 2024-11-15T17:39:58,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/37288ac2e5bc40689dc40f4540eac24d, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:39:58,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/02c6cb35b57d471cbb85a99f35f7cc2e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c6cb35b57d471cbb85a99f35f7cc2e 2024-11-15T17:39:58,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c6cb35b57d471cbb85a99f35f7cc2e, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:39:58,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for ada8f074c978a899b609b6215cb2d474 in 187ms, sequenceid=165, compaction requested=false 2024-11-15T17:39:58,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:58,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T17:39:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:58,911 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:39:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:58,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/70c4437077a74d26b60902eb9e38236a is 50, key is test_row_0/A:col10/1731692398654/Put/seqid=0 2024-11-15T17:39:58,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741873_1049 (size=12151) 2024-11-15T17:39:58,961 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/70c4437077a74d26b60902eb9e38236a 2024-11-15T17:39:58,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:58,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/ef3498c639b04db8b2d46a25d4859e6a is 50, key is test_row_0/B:col10/1731692398654/Put/seqid=0 2024-11-15T17:39:58,993 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/c83e8d1e8b3a4a2c9690eb1c383b418c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/c83e8d1e8b3a4a2c9690eb1c383b418c 2024-11-15T17:39:59,011 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into c83e8d1e8b3a4a2c9690eb1c383b418c(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:59,011 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:59,011 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=13, startTime=1731692398488; duration=0sec 2024-11-15T17:39:59,012 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:59,012 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:39:59,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741874_1050 (size=12151) 2024-11-15T17:39:59,018 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/ef3498c639b04db8b2d46a25d4859e6a 2024-11-15T17:39:59,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/80a29eaa71c54873bda54b131b78dc2e is 50, key is test_row_0/C:col10/1731692398654/Put/seqid=0 2024-11-15T17:39:59,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692459057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692459059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692459060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692459061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692459068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,077 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/997af595ee794c1098deeba3e732551b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/997af595ee794c1098deeba3e732551b 2024-11-15T17:39:59,087 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into 997af595ee794c1098deeba3e732551b(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:39:59,087 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:59,087 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=13, startTime=1731692398489; duration=0sec 2024-11-15T17:39:59,087 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:39:59,088 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:39:59,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741875_1051 (size=12151) 2024-11-15T17:39:59,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692459176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692459176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692459176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692459177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692459186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692459387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692459387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692459387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692459387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692459397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,500 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/80a29eaa71c54873bda54b131b78dc2e 2024-11-15T17:39:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/70c4437077a74d26b60902eb9e38236a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/70c4437077a74d26b60902eb9e38236a 2024-11-15T17:39:59,521 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/70c4437077a74d26b60902eb9e38236a, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T17:39:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/ef3498c639b04db8b2d46a25d4859e6a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ef3498c639b04db8b2d46a25d4859e6a 2024-11-15T17:39:59,538 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ef3498c639b04db8b2d46a25d4859e6a, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T17:39:59,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/80a29eaa71c54873bda54b131b78dc2e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/80a29eaa71c54873bda54b131b78dc2e 2024-11-15T17:39:59,553 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/80a29eaa71c54873bda54b131b78dc2e, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T17:39:59,561 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ada8f074c978a899b609b6215cb2d474 in 650ms, sequenceid=177, compaction requested=true 2024-11-15T17:39:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:39:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T17:39:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T17:39:59,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T17:39:59,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 962 msec 2024-11-15T17:39:59,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 971 msec 2024-11-15T17:39:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:39:59,700 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:39:59,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T17:39:59,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/56c71642b92c4ba698d4d8898a599ffb is 50, key is test_row_0/A:col10/1731692399697/Put/seqid=0 2024-11-15T17:39:59,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692459710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692459716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692459716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692459718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692459724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741876_1052 (size=14541) 2024-11-15T17:39:59,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/56c71642b92c4ba698d4d8898a599ffb 2024-11-15T17:39:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:59,736 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:39:59,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:39:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T17:39:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T17:39:59,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:39:59,743 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:39:59,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:39:59,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/35164e5e48fe4b04ab865f6039837768 is 50, key is test_row_0/B:col10/1731692399697/Put/seqid=0 2024-11-15T17:39:59,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741877_1053 (size=12151) 2024-11-15T17:39:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692459826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692459826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692459836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692459836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:39:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692459836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:39:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:39:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:39:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T17:39:59,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T17:39:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:39:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:39:59,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:59,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692460036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T17:40:00,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692460046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692460047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692460047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692460047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T17:40:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/35164e5e48fe4b04ab865f6039837768 2024-11-15T17:40:00,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/8088884a47c3402a983217b0cabaa374 is 50, key is test_row_0/C:col10/1731692399697/Put/seqid=0 2024-11-15T17:40:00,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T17:40:00,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:00,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741878_1054 (size=12151) 2024-11-15T17:40:00,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/8088884a47c3402a983217b0cabaa374 2024-11-15T17:40:00,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/56c71642b92c4ba698d4d8898a599ffb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/56c71642b92c4ba698d4d8898a599ffb 2024-11-15T17:40:00,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/56c71642b92c4ba698d4d8898a599ffb, entries=200, sequenceid=206, filesize=14.2 K 2024-11-15T17:40:00,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/35164e5e48fe4b04ab865f6039837768 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/35164e5e48fe4b04ab865f6039837768 2024-11-15T17:40:00,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/35164e5e48fe4b04ab865f6039837768, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T17:40:00,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/8088884a47c3402a983217b0cabaa374 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8088884a47c3402a983217b0cabaa374 2024-11-15T17:40:00,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8088884a47c3402a983217b0cabaa374, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T17:40:00,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for ada8f074c978a899b609b6215cb2d474 in 626ms, sequenceid=206, compaction requested=true 2024-11-15T17:40:00,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:00,327 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:00,329 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53676 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:00,329 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:00,329 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:00,329 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:00,329 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5266f13997ee43cfa5973b11a1704d04, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d152b0f7343048a9905836bb6c16a0de, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/70c4437077a74d26b60902eb9e38236a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/56c71642b92c4ba698d4d8898a599ffb] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=52.4 K 2024-11-15T17:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:00,330 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5266f13997ee43cfa5973b11a1704d04, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731692396062 2024-11-15T17:40:00,331 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d152b0f7343048a9905836bb6c16a0de, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731692398302 2024-11-15T17:40:00,331 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48896 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:00,331 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:40:00,331 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,332 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/c83e8d1e8b3a4a2c9690eb1c383b418c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/37288ac2e5bc40689dc40f4540eac24d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ef3498c639b04db8b2d46a25d4859e6a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/35164e5e48fe4b04ab865f6039837768] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=47.8 K 2024-11-15T17:40:00,332 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c83e8d1e8b3a4a2c9690eb1c383b418c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731692396062 2024-11-15T17:40:00,333 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c4437077a74d26b60902eb9e38236a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731692398651 2024-11-15T17:40:00,333 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 37288ac2e5bc40689dc40f4540eac24d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731692398302 2024-11-15T17:40:00,334 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56c71642b92c4ba698d4d8898a599ffb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731692399065 2024-11-15T17:40:00,334 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting ef3498c639b04db8b2d46a25d4859e6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731692398651 2024-11-15T17:40:00,335 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 35164e5e48fe4b04ab865f6039837768, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731692399697 2024-11-15T17:40:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:00,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:40:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T17:40:00,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T17:40:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:00,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,369 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:00,370 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/5a59e24ece3945a18720e3ab55549f79 is 50, key is test_row_0/A:col10/1731692399697/Put/seqid=0 2024-11-15T17:40:00,399 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:00,400 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/3f5a4fe682da4ca19899fe4edc7c4462 is 50, key is test_row_0/B:col10/1731692399697/Put/seqid=0 2024-11-15T17:40:00,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/47f774ebd89d4e9590b51138fe722936 is 50, key is test_row_0/A:col10/1731692399713/Put/seqid=0 2024-11-15T17:40:00,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741880_1056 (size=12629) 2024-11-15T17:40:00,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692460430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692460437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692460437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692460438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692460441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741879_1055 (size=12629) 2024-11-15T17:40:00,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741881_1057 (size=14541) 2024-11-15T17:40:00,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/47f774ebd89d4e9590b51138fe722936 2024-11-15T17:40:00,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8daf476c1bfa433c8f71ae0c850ad87e is 50, key is test_row_0/B:col10/1731692399713/Put/seqid=0 2024-11-15T17:40:00,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741882_1058 (size=12151) 2024-11-15T17:40:00,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T17:40:00,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:00,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8daf476c1bfa433c8f71ae0c850ad87e 2024-11-15T17:40:00,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692460545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692460545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692460545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/6cd1860f828040c8b1e022ce2b652bfa is 50, key is test_row_0/C:col10/1731692399713/Put/seqid=0 2024-11-15T17:40:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692460556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692460556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741883_1059 (size=12151) 2024-11-15T17:40:00,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/6cd1860f828040c8b1e022ce2b652bfa 2024-11-15T17:40:00,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/47f774ebd89d4e9590b51138fe722936 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/47f774ebd89d4e9590b51138fe722936 2024-11-15T17:40:00,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/47f774ebd89d4e9590b51138fe722936, entries=200, sequenceid=218, filesize=14.2 K 2024-11-15T17:40:00,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8daf476c1bfa433c8f71ae0c850ad87e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8daf476c1bfa433c8f71ae0c850ad87e 2024-11-15T17:40:00,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8daf476c1bfa433c8f71ae0c850ad87e, entries=150, sequenceid=218, filesize=11.9 K 2024-11-15T17:40:00,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/6cd1860f828040c8b1e022ce2b652bfa as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6cd1860f828040c8b1e022ce2b652bfa 2024-11-15T17:40:00,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6cd1860f828040c8b1e022ce2b652bfa, entries=150, sequenceid=218, filesize=11.9 K 2024-11-15T17:40:00,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ada8f074c978a899b609b6215cb2d474 in 303ms, sequenceid=218, compaction requested=true 2024-11-15T17:40:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T17:40:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T17:40:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T17:40:00,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T17:40:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,677 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:40:00,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/85e02b055f834df0b3a47dc25541e35f is 50, key is test_row_0/A:col10/1731692400431/Put/seqid=0 2024-11-15T17:40:00,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741884_1060 (size=12151) 2024-11-15T17:40:00,713 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/85e02b055f834df0b3a47dc25541e35f 2024-11-15T17:40:00,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/bd64c66114b249d087b6a74eea17bd2f is 50, key is test_row_0/B:col10/1731692400431/Put/seqid=0 2024-11-15T17:40:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:00,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:00,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741885_1061 (size=12151) 2024-11-15T17:40:00,777 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/bd64c66114b249d087b6a74eea17bd2f 2024-11-15T17:40:00,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692460774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692460775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692460784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692460781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692460784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7de9ca0ee9b14494b41c738f080576e9 is 50, key is test_row_0/C:col10/1731692400431/Put/seqid=0 2024-11-15T17:40:00,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741886_1062 (size=12151) 2024-11-15T17:40:00,847 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/3f5a4fe682da4ca19899fe4edc7c4462 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/3f5a4fe682da4ca19899fe4edc7c4462 2024-11-15T17:40:00,858 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into 3f5a4fe682da4ca19899fe4edc7c4462(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:00,858 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:00,858 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=12, startTime=1731692400329; duration=0sec 2024-11-15T17:40:00,859 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T17:40:00,859 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:00,859 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-15T17:40:00,860 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:40:00,860 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:40:00,860 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. because compaction request was cancelled 2024-11-15T17:40:00,860 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:40:00,861 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T17:40:00,863 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/5a59e24ece3945a18720e3ab55549f79 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5a59e24ece3945a18720e3ab55549f79 2024-11-15T17:40:00,863 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61047 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T17:40:00,864 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:40:00,864 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:00,864 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/997af595ee794c1098deeba3e732551b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c6cb35b57d471cbb85a99f35f7cc2e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/80a29eaa71c54873bda54b131b78dc2e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8088884a47c3402a983217b0cabaa374, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6cd1860f828040c8b1e022ce2b652bfa] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=59.6 K 2024-11-15T17:40:00,865 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 997af595ee794c1098deeba3e732551b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731692396062 2024-11-15T17:40:00,866 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c6cb35b57d471cbb85a99f35f7cc2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731692398302 2024-11-15T17:40:00,866 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a29eaa71c54873bda54b131b78dc2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731692398651 2024-11-15T17:40:00,867 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8088884a47c3402a983217b0cabaa374, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731692399697 2024-11-15T17:40:00,867 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd1860f828040c8b1e022ce2b652bfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692399713 2024-11-15T17:40:00,872 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into 5a59e24ece3945a18720e3ab55549f79(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:00,872 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:00,872 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=12, startTime=1731692400327; duration=0sec 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. because compaction request was cancelled 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:40:00,873 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T17:40:00,874 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:40:00,874 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:40:00,874 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. because compaction request was cancelled 2024-11-15T17:40:00,875 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T17:40:00,891 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:00,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/33be76fc0c4f44e18b797a68caeda054 is 50, key is test_row_0/C:col10/1731692399713/Put/seqid=0 2024-11-15T17:40:00,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692460896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741887_1063 (size=12663) 2024-11-15T17:40:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692460896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692460896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692460897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692460897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:00,929 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/33be76fc0c4f44e18b797a68caeda054 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/33be76fc0c4f44e18b797a68caeda054 2024-11-15T17:40:00,937 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into 33be76fc0c4f44e18b797a68caeda054(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:00,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:00,938 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=11, startTime=1731692400657; duration=0sec 2024-11-15T17:40:00,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:00,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:40:01,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692461106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692461130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692461131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692461131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692461131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,218 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7de9ca0ee9b14494b41c738f080576e9 2024-11-15T17:40:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/85e02b055f834df0b3a47dc25541e35f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/85e02b055f834df0b3a47dc25541e35f 2024-11-15T17:40:01,236 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/85e02b055f834df0b3a47dc25541e35f, entries=150, sequenceid=242, filesize=11.9 K 2024-11-15T17:40:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/bd64c66114b249d087b6a74eea17bd2f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/bd64c66114b249d087b6a74eea17bd2f 2024-11-15T17:40:01,246 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/bd64c66114b249d087b6a74eea17bd2f, entries=150, sequenceid=242, filesize=11.9 K 2024-11-15T17:40:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7de9ca0ee9b14494b41c738f080576e9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7de9ca0ee9b14494b41c738f080576e9 2024-11-15T17:40:01,255 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7de9ca0ee9b14494b41c738f080576e9, entries=150, sequenceid=242, filesize=11.9 K 2024-11-15T17:40:01,257 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ada8f074c978a899b609b6215cb2d474 in 580ms, sequenceid=242, compaction requested=true 2024-11-15T17:40:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T17:40:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T17:40:01,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T17:40:01,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-11-15T17:40:01,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5270 sec 2024-11-15T17:40:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:01,420 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:01,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:40:01,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/060a0083db1140b9816286e9da2875a0 is 50, key is test_row_0/A:col10/1731692401418/Put/seqid=0 2024-11-15T17:40:01,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692461475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692461475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741888_1064 (size=12201) 2024-11-15T17:40:01,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692461475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692461476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692461476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/060a0083db1140b9816286e9da2875a0 2024-11-15T17:40:01,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/4d95968586484d17b13d21d4fb1883c2 is 50, key is test_row_0/B:col10/1731692401418/Put/seqid=0 2024-11-15T17:40:01,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741889_1065 (size=12201) 2024-11-15T17:40:01,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692461586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692461586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692461586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692461586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692461586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692461796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692461796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692461797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692461797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692461805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T17:40:01,886 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:01,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T17:40:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T17:40:01,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:01,896 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:01,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:01,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/4d95968586484d17b13d21d4fb1883c2 2024-11-15T17:40:01,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/c41333927ec34cd08bd569df32326db7 is 50, key is test_row_0/C:col10/1731692401418/Put/seqid=0 2024-11-15T17:40:01,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741890_1066 (size=12201) 2024-11-15T17:40:01,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/c41333927ec34cd08bd569df32326db7 2024-11-15T17:40:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/060a0083db1140b9816286e9da2875a0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/060a0083db1140b9816286e9da2875a0 2024-11-15T17:40:01,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/060a0083db1140b9816286e9da2875a0, entries=150, sequenceid=258, filesize=11.9 K 2024-11-15T17:40:01,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/4d95968586484d17b13d21d4fb1883c2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/4d95968586484d17b13d21d4fb1883c2 2024-11-15T17:40:01,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/4d95968586484d17b13d21d4fb1883c2, entries=150, sequenceid=258, filesize=11.9 K 2024-11-15T17:40:01,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/c41333927ec34cd08bd569df32326db7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c41333927ec34cd08bd569df32326db7 2024-11-15T17:40:01,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c41333927ec34cd08bd569df32326db7, entries=150, sequenceid=258, filesize=11.9 K 2024-11-15T17:40:01,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ada8f074c978a899b609b6215cb2d474 in 574ms, sequenceid=258, compaction requested=true 2024-11-15T17:40:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:01,994 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:01,994 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T17:40:01,997 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:01,997 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51522 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:01,997 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:40:01,997 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:40:01,997 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:01,997 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:01,997 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5a59e24ece3945a18720e3ab55549f79, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/47f774ebd89d4e9590b51138fe722936, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/85e02b055f834df0b3a47dc25541e35f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/060a0083db1140b9816286e9da2875a0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=50.3 K 2024-11-15T17:40:01,997 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/3f5a4fe682da4ca19899fe4edc7c4462, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8daf476c1bfa433c8f71ae0c850ad87e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/bd64c66114b249d087b6a74eea17bd2f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/4d95968586484d17b13d21d4fb1883c2] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=48.0 K 2024-11-15T17:40:01,998 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5a4fe682da4ca19899fe4edc7c4462, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731692399697 2024-11-15T17:40:01,998 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a59e24ece3945a18720e3ab55549f79, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731692399697 2024-11-15T17:40:01,998 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8daf476c1bfa433c8f71ae0c850ad87e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692399713 2024-11-15T17:40:01,999 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bd64c66114b249d087b6a74eea17bd2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731692400423 2024-11-15T17:40:01,999 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f774ebd89d4e9590b51138fe722936, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692399713 2024-11-15T17:40:02,000 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d95968586484d17b13d21d4fb1883c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692400781 2024-11-15T17:40:02,001 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e02b055f834df0b3a47dc25541e35f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731692400423 2024-11-15T17:40:02,002 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 060a0083db1140b9816286e9da2875a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692400781 2024-11-15T17:40:02,024 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:02,025 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/0e07da88c8664f60a9fef789aed56c2a is 50, key is test_row_0/B:col10/1731692401418/Put/seqid=0 2024-11-15T17:40:02,032 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:02,033 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/3e9e229bc37348e09559b6ab8e0836e1 is 50, key is test_row_0/A:col10/1731692401418/Put/seqid=0 2024-11-15T17:40:02,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T17:40:02,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:02,052 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:40:02,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741891_1067 (size=12815) 2024-11-15T17:40:02,086 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/0e07da88c8664f60a9fef789aed56c2a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/0e07da88c8664f60a9fef789aed56c2a 2024-11-15T17:40:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/d348027823be4173926c0caaae6652f9 is 50, key is test_row_0/A:col10/1731692401474/Put/seqid=0 2024-11-15T17:40:02,105 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into 0e07da88c8664f60a9fef789aed56c2a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:02,105 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:02,107 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=12, startTime=1731692401994; duration=0sec 2024-11-15T17:40:02,107 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:02,108 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:02,108 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:02,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741892_1068 (size=12815) 2024-11-15T17:40:02,110 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:02,110 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:40:02,110 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:02,110 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/33be76fc0c4f44e18b797a68caeda054, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7de9ca0ee9b14494b41c738f080576e9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c41333927ec34cd08bd569df32326db7] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=36.1 K 2024-11-15T17:40:02,111 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 33be76fc0c4f44e18b797a68caeda054, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692399713 2024-11-15T17:40:02,112 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de9ca0ee9b14494b41c738f080576e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731692400423 2024-11-15T17:40:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:02,113 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c41333927ec34cd08bd569df32326db7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692400781 2024-11-15T17:40:02,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741893_1069 (size=12301) 2024-11-15T17:40:02,130 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/3e9e229bc37348e09559b6ab8e0836e1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3e9e229bc37348e09559b6ab8e0836e1 2024-11-15T17:40:02,131 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/d348027823be4173926c0caaae6652f9 2024-11-15T17:40:02,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692462129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692462133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692462133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692462139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692462139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,155 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into 3e9e229bc37348e09559b6ab8e0836e1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:02,155 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:02,156 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=12, startTime=1731692401994; duration=0sec 2024-11-15T17:40:02,156 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:02,156 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:40:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/40bda268beac4880b1e4a867d0662de4 is 50, key is test_row_0/B:col10/1731692401474/Put/seqid=0 2024-11-15T17:40:02,161 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:02,162 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/fd1185870d474d55bec05f30f6965338 is 50, key is test_row_0/C:col10/1731692401418/Put/seqid=0 2024-11-15T17:40:02,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741894_1070 (size=12301) 2024-11-15T17:40:02,199 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/40bda268beac4880b1e4a867d0662de4 2024-11-15T17:40:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T17:40:02,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741895_1071 (size=12815) 2024-11-15T17:40:02,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/e1e79ed844e04e31a8ee5e628d96f3bb is 50, key is test_row_0/C:col10/1731692401474/Put/seqid=0 2024-11-15T17:40:02,227 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/fd1185870d474d55bec05f30f6965338 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/fd1185870d474d55bec05f30f6965338 2024-11-15T17:40:02,237 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into fd1185870d474d55bec05f30f6965338(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:02,237 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:02,237 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=13, startTime=1731692401994; duration=0sec 2024-11-15T17:40:02,237 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:02,238 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:40:02,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692462246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692462246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692462246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692462246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741896_1072 (size=12301) 2024-11-15T17:40:02,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692462256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692462455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692462456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692462456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692462456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692462465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T17:40:02,654 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/e1e79ed844e04e31a8ee5e628d96f3bb 2024-11-15T17:40:02,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/d348027823be4173926c0caaae6652f9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d348027823be4173926c0caaae6652f9 2024-11-15T17:40:02,671 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d348027823be4173926c0caaae6652f9, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T17:40:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/40bda268beac4880b1e4a867d0662de4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/40bda268beac4880b1e4a867d0662de4 2024-11-15T17:40:02,682 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/40bda268beac4880b1e4a867d0662de4, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T17:40:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/e1e79ed844e04e31a8ee5e628d96f3bb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/e1e79ed844e04e31a8ee5e628d96f3bb 2024-11-15T17:40:02,699 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/e1e79ed844e04e31a8ee5e628d96f3bb, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T17:40:02,701 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ada8f074c978a899b609b6215cb2d474 in 648ms, sequenceid=281, compaction requested=false 2024-11-15T17:40:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T17:40:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T17:40:02,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T17:40:02,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 808 msec 2024-11-15T17:40:02,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 821 msec 2024-11-15T17:40:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:02,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T17:40:02,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/115f639d4aeb49749a942b2c80285671 is 50, key is test_row_0/A:col10/1731692402766/Put/seqid=0 2024-11-15T17:40:02,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741897_1073 (size=14741) 2024-11-15T17:40:02,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/115f639d4aeb49749a942b2c80285671 2024-11-15T17:40:02,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/1c1139fc34e347d8a03c850741ec385b is 50, key is test_row_0/B:col10/1731692402766/Put/seqid=0 2024-11-15T17:40:02,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692462807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692462811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692462815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692462816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692462818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741898_1074 (size=12301) 2024-11-15T17:40:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692462925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692462925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692462925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692462926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692462926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T17:40:03,026 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:03,026 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T17:40:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:40:03,033 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:03,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:03,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:40:03,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692463136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692463136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692463136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692463137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692463137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T17:40:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:03,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:03,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:03,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/1c1139fc34e347d8a03c850741ec385b 2024-11-15T17:40:03,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/c4d1079d55464cda98bd3db409d10f60 is 50, key is test_row_0/C:col10/1731692402766/Put/seqid=0 2024-11-15T17:40:03,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741899_1075 (size=12301) 2024-11-15T17:40:03,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/c4d1079d55464cda98bd3db409d10f60 2024-11-15T17:40:03,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/115f639d4aeb49749a942b2c80285671 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/115f639d4aeb49749a942b2c80285671 2024-11-15T17:40:03,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/115f639d4aeb49749a942b2c80285671, entries=200, sequenceid=300, filesize=14.4 K 2024-11-15T17:40:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/1c1139fc34e347d8a03c850741ec385b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1c1139fc34e347d8a03c850741ec385b 2024-11-15T17:40:03,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1c1139fc34e347d8a03c850741ec385b, entries=150, sequenceid=300, filesize=12.0 K 2024-11-15T17:40:03,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/c4d1079d55464cda98bd3db409d10f60 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c4d1079d55464cda98bd3db409d10f60 2024-11-15T17:40:03,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c4d1079d55464cda98bd3db409d10f60, entries=150, sequenceid=300, filesize=12.0 K 2024-11-15T17:40:03,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ada8f074c978a899b609b6215cb2d474 in 518ms, sequenceid=300, compaction requested=true 2024-11-15T17:40:03,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:03,290 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:03,290 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:03,291 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:03,291 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:40:03,291 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:03,291 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:03,291 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:40:03,291 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:03,291 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3e9e229bc37348e09559b6ab8e0836e1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d348027823be4173926c0caaae6652f9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/115f639d4aeb49749a942b2c80285671] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=38.9 K 2024-11-15T17:40:03,291 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/0e07da88c8664f60a9fef789aed56c2a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/40bda268beac4880b1e4a867d0662de4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1c1139fc34e347d8a03c850741ec385b] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=36.5 K 2024-11-15T17:40:03,292 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e07da88c8664f60a9fef789aed56c2a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692400781 2024-11-15T17:40:03,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e9e229bc37348e09559b6ab8e0836e1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692400781 2024-11-15T17:40:03,292 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 40bda268beac4880b1e4a867d0662de4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731692401460 2024-11-15T17:40:03,293 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d348027823be4173926c0caaae6652f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731692401460 2024-11-15T17:40:03,293 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c1139fc34e347d8a03c850741ec385b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731692402137 2024-11-15T17:40:03,294 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 115f639d4aeb49749a942b2c80285671, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731692402137 2024-11-15T17:40:03,319 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:03,319 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:03,319 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/4953350560dd4598844a1eab618b1234 is 50, key is test_row_0/A:col10/1731692402766/Put/seqid=0 2024-11-15T17:40:03,319 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/892ade4f6b0a41a7864d25d87b681f64 is 50, key is test_row_0/B:col10/1731692402766/Put/seqid=0 2024-11-15T17:40:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:40:03,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741900_1076 (size=13017) 2024-11-15T17:40:03,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741901_1077 (size=13017) 2024-11-15T17:40:03,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T17:40:03,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:03,358 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:40:03,359 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/4953350560dd4598844a1eab618b1234 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/4953350560dd4598844a1eab618b1234 2024-11-15T17:40:03,361 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/892ade4f6b0a41a7864d25d87b681f64 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/892ade4f6b0a41a7864d25d87b681f64 2024-11-15T17:40:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/8f6e9350a093418a9e0cc9f90fdd1a70 is 50, key is test_row_0/A:col10/1731692402814/Put/seqid=0 2024-11-15T17:40:03,370 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into 4953350560dd4598844a1eab618b1234(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:03,370 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:03,370 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=13, startTime=1731692403289; duration=0sec 2024-11-15T17:40:03,371 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:03,371 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:40:03,371 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:03,372 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:03,372 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:40:03,373 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:03,373 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/fd1185870d474d55bec05f30f6965338, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/e1e79ed844e04e31a8ee5e628d96f3bb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c4d1079d55464cda98bd3db409d10f60] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=36.5 K 2024-11-15T17:40:03,374 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1185870d474d55bec05f30f6965338, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692400781 2024-11-15T17:40:03,377 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e79ed844e04e31a8ee5e628d96f3bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731692401460 2024-11-15T17:40:03,378 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d1079d55464cda98bd3db409d10f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731692402137 2024-11-15T17:40:03,382 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into 892ade4f6b0a41a7864d25d87b681f64(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:03,382 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:03,383 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=13, startTime=1731692403290; duration=0sec 2024-11-15T17:40:03,383 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:03,385 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:03,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741902_1078 (size=12301) 2024-11-15T17:40:03,391 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/8f6e9350a093418a9e0cc9f90fdd1a70 2024-11-15T17:40:03,405 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#66 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:03,407 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/6b1c80d3bf6342429e8d17ad6597f96f is 50, key is test_row_0/C:col10/1731692402766/Put/seqid=0 2024-11-15T17:40:03,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/13af6dbd36e34720857fb9d2cd65f911 is 50, key is test_row_0/B:col10/1731692402814/Put/seqid=0 2024-11-15T17:40:03,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741903_1079 (size=13017) 2024-11-15T17:40:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:03,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741904_1080 (size=12301) 2024-11-15T17:40:03,459 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/13af6dbd36e34720857fb9d2cd65f911 2024-11-15T17:40:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/3c567255313a4b558e993f3856ee8c0b is 50, key is test_row_0/C:col10/1731692402814/Put/seqid=0 2024-11-15T17:40:03,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692463471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692463477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692463477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692463482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692463482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741905_1081 (size=12301) 2024-11-15T17:40:03,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692463585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692463596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692463596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692463596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692463596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:40:03,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692463795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692463806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692463806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692463806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692463806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:03,852 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/6b1c80d3bf6342429e8d17ad6597f96f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6b1c80d3bf6342429e8d17ad6597f96f 2024-11-15T17:40:03,863 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into 6b1c80d3bf6342429e8d17ad6597f96f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:03,863 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:03,863 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=13, startTime=1731692403290; duration=0sec 2024-11-15T17:40:03,863 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:03,863 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:40:03,919 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/3c567255313a4b558e993f3856ee8c0b 2024-11-15T17:40:03,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/8f6e9350a093418a9e0cc9f90fdd1a70 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/8f6e9350a093418a9e0cc9f90fdd1a70 2024-11-15T17:40:03,936 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/8f6e9350a093418a9e0cc9f90fdd1a70, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T17:40:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/13af6dbd36e34720857fb9d2cd65f911 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/13af6dbd36e34720857fb9d2cd65f911 2024-11-15T17:40:03,945 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/13af6dbd36e34720857fb9d2cd65f911, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T17:40:03,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/3c567255313a4b558e993f3856ee8c0b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/3c567255313a4b558e993f3856ee8c0b 2024-11-15T17:40:03,954 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/3c567255313a4b558e993f3856ee8c0b, entries=150, sequenceid=320, filesize=12.0 K 2024-11-15T17:40:03,955 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for ada8f074c978a899b609b6215cb2d474 in 598ms, sequenceid=320, compaction requested=false 2024-11-15T17:40:03,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:03,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:03,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T17:40:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T17:40:03,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T17:40:03,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 921 msec 2024-11-15T17:40:03,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 936 msec 2024-11-15T17:40:04,111 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:04,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:40:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:04,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/0255ccf9cdda48eb887ec4d24c07b230 is 50, key is test_row_0/A:col10/1731692403476/Put/seqid=0 2024-11-15T17:40:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692464148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692464150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692464151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692464151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741906_1082 (size=14741) 2024-11-15T17:40:04,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/0255ccf9cdda48eb887ec4d24c07b230 2024-11-15T17:40:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692464153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:40:04,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:04,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T17:40:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T17:40:04,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:04,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:04,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:04,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8eccbfa9b2cf464a96bd0ae2172ef522 is 50, key is test_row_0/B:col10/1731692403476/Put/seqid=0 2024-11-15T17:40:04,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741907_1083 (size=12301) 2024-11-15T17:40:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692464265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692464265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692464265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692464266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T17:40:04,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692464289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T17:40:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692464476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692464476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692464476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692464477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T17:40:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:04,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T17:40:04,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692464495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8eccbfa9b2cf464a96bd0ae2172ef522 2024-11-15T17:40:04,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/5ba57c0126584335afaf7915b11f596f is 50, key is test_row_0/C:col10/1731692403476/Put/seqid=0 2024-11-15T17:40:04,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741908_1084 (size=12301) 2024-11-15T17:40:04,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T17:40:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T17:40:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692464785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692464785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692464785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692464785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T17:40:04,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692464806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:04,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T17:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:04,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:05,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/5ba57c0126584335afaf7915b11f596f 2024-11-15T17:40:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/0255ccf9cdda48eb887ec4d24c07b230 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/0255ccf9cdda48eb887ec4d24c07b230 2024-11-15T17:40:05,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/0255ccf9cdda48eb887ec4d24c07b230, entries=200, sequenceid=340, filesize=14.4 K 2024-11-15T17:40:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8eccbfa9b2cf464a96bd0ae2172ef522 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8eccbfa9b2cf464a96bd0ae2172ef522 2024-11-15T17:40:05,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8eccbfa9b2cf464a96bd0ae2172ef522, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T17:40:05,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/5ba57c0126584335afaf7915b11f596f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/5ba57c0126584335afaf7915b11f596f 2024-11-15T17:40:05,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/5ba57c0126584335afaf7915b11f596f, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T17:40:05,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ada8f074c978a899b609b6215cb2d474 in 948ms, sequenceid=340, compaction requested=true 2024-11-15T17:40:05,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:05,059 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:05,060 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:05,062 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:05,062 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:40:05,062 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:05,062 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/4953350560dd4598844a1eab618b1234, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/8f6e9350a093418a9e0cc9f90fdd1a70, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/0255ccf9cdda48eb887ec4d24c07b230] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=39.1 K 2024-11-15T17:40:05,063 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:05,063 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:40:05,063 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:05,063 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/892ade4f6b0a41a7864d25d87b681f64, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/13af6dbd36e34720857fb9d2cd65f911, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8eccbfa9b2cf464a96bd0ae2172ef522] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=36.7 K 2024-11-15T17:40:05,065 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4953350560dd4598844a1eab618b1234, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731692402137 2024-11-15T17:40:05,065 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 892ade4f6b0a41a7864d25d87b681f64, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731692402137 2024-11-15T17:40:05,065 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f6e9350a093418a9e0cc9f90fdd1a70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731692402802 2024-11-15T17:40:05,066 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 13af6dbd36e34720857fb9d2cd65f911, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731692402802 2024-11-15T17:40:05,066 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0255ccf9cdda48eb887ec4d24c07b230, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731692403474 2024-11-15T17:40:05,066 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eccbfa9b2cf464a96bd0ae2172ef522, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731692403476 2024-11-15T17:40:05,080 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:05,081 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/b1f238477ce54a8ea7cf3713bd17a9da is 50, key is test_row_0/B:col10/1731692403476/Put/seqid=0 2024-11-15T17:40:05,088 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:05,088 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/1b18d43897f74d36931c5c3483857e8c is 50, key is test_row_0/A:col10/1731692403476/Put/seqid=0 2024-11-15T17:40:05,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T17:40:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:05,103 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T17:40:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/61134c005d6e47b8bbafbb8f0e0056ab is 50, key is test_row_0/A:col10/1731692404149/Put/seqid=0 2024-11-15T17:40:05,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741910_1086 (size=13119) 2024-11-15T17:40:05,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741909_1085 (size=13119) 2024-11-15T17:40:05,152 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/1b18d43897f74d36931c5c3483857e8c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b18d43897f74d36931c5c3483857e8c 2024-11-15T17:40:05,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741911_1087 (size=12301) 2024-11-15T17:40:05,165 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into 1b18d43897f74d36931c5c3483857e8c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:05,165 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:05,165 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=13, startTime=1731692405059; duration=0sec 2024-11-15T17:40:05,165 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:05,165 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:40:05,165 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:05,167 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:05,167 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:40:05,167 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:05,167 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6b1c80d3bf6342429e8d17ad6597f96f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/3c567255313a4b558e993f3856ee8c0b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/5ba57c0126584335afaf7915b11f596f] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=36.7 K 2024-11-15T17:40:05,168 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1c80d3bf6342429e8d17ad6597f96f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731692402137 2024-11-15T17:40:05,169 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c567255313a4b558e993f3856ee8c0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731692402802 2024-11-15T17:40:05,169 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba57c0126584335afaf7915b11f596f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731692403476 2024-11-15T17:40:05,183 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:05,183 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7bbaa1d5aed84f78b532d9b1dc536633 is 50, key is test_row_0/C:col10/1731692403476/Put/seqid=0 2024-11-15T17:40:05,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741912_1088 (size=13119) 2024-11-15T17:40:05,203 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7bbaa1d5aed84f78b532d9b1dc536633 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7bbaa1d5aed84f78b532d9b1dc536633 2024-11-15T17:40:05,216 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into 7bbaa1d5aed84f78b532d9b1dc536633(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:05,217 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:05,217 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=13, startTime=1731692405060; duration=0sec 2024-11-15T17:40:05,217 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:05,217 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:40:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T17:40:05,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692465320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692465321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692465324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692465326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692465326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692465436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692465436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692465445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692465445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692465445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,562 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/b1f238477ce54a8ea7cf3713bd17a9da as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b1f238477ce54a8ea7cf3713bd17a9da 2024-11-15T17:40:05,562 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/61134c005d6e47b8bbafbb8f0e0056ab 2024-11-15T17:40:05,575 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into b1f238477ce54a8ea7cf3713bd17a9da(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:05,576 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:05,576 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=13, startTime=1731692405060; duration=0sec 2024-11-15T17:40:05,578 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:05,578 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:05,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/ac53afb1a0174a80b5edc071373c8fdf is 50, key is test_row_0/B:col10/1731692404149/Put/seqid=0 2024-11-15T17:40:05,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741913_1089 (size=12301) 2024-11-15T17:40:05,594 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/ac53afb1a0174a80b5edc071373c8fdf 2024-11-15T17:40:05,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/1e5dd0568b1d4e27a04d04b238f0fcc0 is 50, key is test_row_0/C:col10/1731692404149/Put/seqid=0 2024-11-15T17:40:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741914_1090 (size=12301) 2024-11-15T17:40:05,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692465645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692465645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692465656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692465656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692465656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692465955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692465956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692465965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692465965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692465965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:05,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,019 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/1e5dd0568b1d4e27a04d04b238f0fcc0 2024-11-15T17:40:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/61134c005d6e47b8bbafbb8f0e0056ab as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61134c005d6e47b8bbafbb8f0e0056ab 2024-11-15T17:40:06,044 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61134c005d6e47b8bbafbb8f0e0056ab, entries=150, sequenceid=359, filesize=12.0 K 2024-11-15T17:40:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/ac53afb1a0174a80b5edc071373c8fdf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ac53afb1a0174a80b5edc071373c8fdf 2024-11-15T17:40:06,067 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ac53afb1a0174a80b5edc071373c8fdf, entries=150, sequenceid=359, filesize=12.0 K 2024-11-15T17:40:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/1e5dd0568b1d4e27a04d04b238f0fcc0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1e5dd0568b1d4e27a04d04b238f0fcc0 2024-11-15T17:40:06,077 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1e5dd0568b1d4e27a04d04b238f0fcc0, entries=150, sequenceid=359, filesize=12.0 K 2024-11-15T17:40:06,078 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ada8f074c978a899b609b6215cb2d474 in 975ms, sequenceid=359, compaction requested=false 2024-11-15T17:40:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T17:40:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T17:40:06,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T17:40:06,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9100 sec 2024-11-15T17:40:06,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9210 sec 2024-11-15T17:40:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T17:40:06,326 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:06,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-15T17:40:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T17:40:06,330 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:06,331 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:06,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T17:40:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:06,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:06,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T17:40:06,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/e9a82b346a8c47ddb713a4be4e2f276d is 50, key is test_row_0/A:col10/1731692406468/Put/seqid=0 2024-11-15T17:40:06,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741915_1091 (size=17181) 2024-11-15T17:40:06,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692466496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/e9a82b346a8c47ddb713a4be4e2f276d 2024-11-15T17:40:06,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692466503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692466504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692466504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692466508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/542e972f1b52484fbb032aba79a5fa64 is 50, key is test_row_0/B:col10/1731692406468/Put/seqid=0 2024-11-15T17:40:06,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741916_1092 (size=12301) 2024-11-15T17:40:06,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/542e972f1b52484fbb032aba79a5fa64 2024-11-15T17:40:06,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/687713a17f714bdfa3f95140fb685574 is 50, key is test_row_0/C:col10/1731692406468/Put/seqid=0 2024-11-15T17:40:06,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741917_1093 (size=12301) 2024-11-15T17:40:06,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/687713a17f714bdfa3f95140fb685574 2024-11-15T17:40:06,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/e9a82b346a8c47ddb713a4be4e2f276d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e9a82b346a8c47ddb713a4be4e2f276d 2024-11-15T17:40:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692466606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e9a82b346a8c47ddb713a4be4e2f276d, entries=250, sequenceid=381, filesize=16.8 K 2024-11-15T17:40:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/542e972f1b52484fbb032aba79a5fa64 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/542e972f1b52484fbb032aba79a5fa64 2024-11-15T17:40:06,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/542e972f1b52484fbb032aba79a5fa64, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T17:40:06,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692466619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692466619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692466619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/687713a17f714bdfa3f95140fb685574 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/687713a17f714bdfa3f95140fb685574 2024-11-15T17:40:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/687713a17f714bdfa3f95140fb685574, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T17:40:06,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for ada8f074c978a899b609b6215cb2d474 in 162ms, sequenceid=381, compaction requested=true 2024-11-15T17:40:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:06,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:06,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:06,634 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:06,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:06,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:06,634 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:06,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:06,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:06,636 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:06,636 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:06,636 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:40:06,636 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:40:06,636 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,636 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,636 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b18d43897f74d36931c5c3483857e8c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61134c005d6e47b8bbafbb8f0e0056ab, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e9a82b346a8c47ddb713a4be4e2f276d] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=41.6 K 2024-11-15T17:40:06,636 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b1f238477ce54a8ea7cf3713bd17a9da, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ac53afb1a0174a80b5edc071373c8fdf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/542e972f1b52484fbb032aba79a5fa64] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=36.8 K 2024-11-15T17:40:06,638 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f238477ce54a8ea7cf3713bd17a9da, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731692403476 2024-11-15T17:40:06,638 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b18d43897f74d36931c5c3483857e8c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731692403476 2024-11-15T17:40:06,638 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61134c005d6e47b8bbafbb8f0e0056ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731692404149 2024-11-15T17:40:06,638 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting ac53afb1a0174a80b5edc071373c8fdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731692404149 2024-11-15T17:40:06,639 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 542e972f1b52484fbb032aba79a5fa64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731692405322 2024-11-15T17:40:06,639 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a82b346a8c47ddb713a4be4e2f276d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731692405322 2024-11-15T17:40:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:06,642 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:06,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T17:40:06,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T17:40:06,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/b5ce5c0fd10a4644bd1bc2be10094884 is 50, key is test_row_0/A:col10/1731692406640/Put/seqid=0 2024-11-15T17:40:06,659 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:06,660 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/abcb2c23641146f1b8443b4d84f0254c is 50, key is test_row_0/B:col10/1731692406468/Put/seqid=0 2024-11-15T17:40:06,662 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:06,662 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/61592f64fa704afcaa660224c24a9cc0 is 50, key is test_row_0/A:col10/1731692406468/Put/seqid=0 2024-11-15T17:40:06,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741918_1094 (size=14741) 2024-11-15T17:40:06,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741919_1095 (size=13221) 2024-11-15T17:40:06,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741920_1096 (size=13221) 2024-11-15T17:40:06,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692466768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692466815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692466825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692466826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692466836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692466875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:06,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:06,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T17:40:07,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/b5ce5c0fd10a4644bd1bc2be10094884 2024-11-15T17:40:07,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692467085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/b7ba9dc054ed42d69361439ee719907a is 50, key is test_row_0/B:col10/1731692406640/Put/seqid=0 2024-11-15T17:40:07,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741921_1097 (size=12301) 2024-11-15T17:40:07,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692467125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,135 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/abcb2c23641146f1b8443b4d84f0254c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/abcb2c23641146f1b8443b4d84f0254c 2024-11-15T17:40:07,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692467136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692467136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,145 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into abcb2c23641146f1b8443b4d84f0254c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:07,145 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:07,145 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=13, startTime=1731692406634; duration=0sec 2024-11-15T17:40:07,145 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:07,145 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:07,145 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:07,146 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:07,146 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:40:07,147 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,147 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7bbaa1d5aed84f78b532d9b1dc536633, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1e5dd0568b1d4e27a04d04b238f0fcc0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/687713a17f714bdfa3f95140fb685574] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=36.8 K 2024-11-15T17:40:07,147 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bbaa1d5aed84f78b532d9b1dc536633, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731692403476 2024-11-15T17:40:07,148 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5dd0568b1d4e27a04d04b238f0fcc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731692404149 2024-11-15T17:40:07,148 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 687713a17f714bdfa3f95140fb685574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731692405322 2024-11-15T17:40:07,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692467145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,160 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/61592f64fa704afcaa660224c24a9cc0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61592f64fa704afcaa660224c24a9cc0 2024-11-15T17:40:07,161 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:07,161 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/02c0977207f8466a9122b093bc599082 is 50, key is test_row_0/C:col10/1731692406468/Put/seqid=0 2024-11-15T17:40:07,168 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into 61592f64fa704afcaa660224c24a9cc0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:07,168 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:07,168 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=13, startTime=1731692406633; duration=0sec 2024-11-15T17:40:07,169 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:07,169 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:40:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741922_1098 (size=13221) 2024-11-15T17:40:07,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692467395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T17:40:07,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/b7ba9dc054ed42d69361439ee719907a 2024-11-15T17:40:07,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/bd3e04ab0b774e9fa60d92bf80b2fb54 is 50, key is test_row_0/C:col10/1731692406640/Put/seqid=0 2024-11-15T17:40:07,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741923_1099 (size=12301) 2024-11-15T17:40:07,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,587 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/02c0977207f8466a9122b093bc599082 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c0977207f8466a9122b093bc599082 2024-11-15T17:40:07,599 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into 02c0977207f8466a9122b093bc599082(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:07,599 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:07,599 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=13, startTime=1731692406634; duration=0sec 2024-11-15T17:40:07,600 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:07,600 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:40:07,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692467636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692467656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692467657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692467666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:07,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:07,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692467906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:07,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/bd3e04ab0b774e9fa60d92bf80b2fb54 2024-11-15T17:40:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/b5ce5c0fd10a4644bd1bc2be10094884 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/b5ce5c0fd10a4644bd1bc2be10094884 2024-11-15T17:40:07,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/b5ce5c0fd10a4644bd1bc2be10094884, entries=200, sequenceid=400, filesize=14.4 K 2024-11-15T17:40:07,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/b7ba9dc054ed42d69361439ee719907a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b7ba9dc054ed42d69361439ee719907a 2024-11-15T17:40:07,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b7ba9dc054ed42d69361439ee719907a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:40:07,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/bd3e04ab0b774e9fa60d92bf80b2fb54 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/bd3e04ab0b774e9fa60d92bf80b2fb54 2024-11-15T17:40:07,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/bd3e04ab0b774e9fa60d92bf80b2fb54, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:40:07,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ada8f074c978a899b609b6215cb2d474 in 1349ms, sequenceid=400, compaction requested=false 2024-11-15T17:40:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:08,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T17:40:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:08,030 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:40:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/3540b0f358aa430b9d536c2966773783 is 50, key is test_row_0/A:col10/1731692406735/Put/seqid=0 2024-11-15T17:40:08,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741924_1100 (size=12301) 2024-11-15T17:40:08,079 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/3540b0f358aa430b9d536c2966773783 2024-11-15T17:40:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/2d2821686357412c9da9af8c17392a5a is 50, key is test_row_0/B:col10/1731692406735/Put/seqid=0 2024-11-15T17:40:08,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741925_1101 (size=12301) 2024-11-15T17:40:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T17:40:08,517 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/2d2821686357412c9da9af8c17392a5a 2024-11-15T17:40:08,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7320942571aa4725b78929813b384b64 is 50, key is test_row_0/C:col10/1731692406735/Put/seqid=0 2024-11-15T17:40:08,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741926_1102 (size=12301) 2024-11-15T17:40:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:08,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. as already flushing 2024-11-15T17:40:08,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692468672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692468675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54430 deadline: 1731692468675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692468676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,683 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in ada8f074c978a899b609b6215cb2d474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692468785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692468785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692468785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731692468916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:08,919 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in ada8f074c978a899b609b6215cb2d474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:08,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:08,941 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7320942571aa4725b78929813b384b64 2024-11-15T17:40:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/3540b0f358aa430b9d536c2966773783 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3540b0f358aa430b9d536c2966773783 2024-11-15T17:40:08,951 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3540b0f358aa430b9d536c2966773783, entries=150, sequenceid=420, filesize=12.0 K 2024-11-15T17:40:08,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/2d2821686357412c9da9af8c17392a5a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2d2821686357412c9da9af8c17392a5a 2024-11-15T17:40:08,957 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2d2821686357412c9da9af8c17392a5a, entries=150, sequenceid=420, filesize=12.0 K 2024-11-15T17:40:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/7320942571aa4725b78929813b384b64 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7320942571aa4725b78929813b384b64 2024-11-15T17:40:08,963 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7320942571aa4725b78929813b384b64, entries=150, sequenceid=420, filesize=12.0 K 2024-11-15T17:40:08,964 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for ada8f074c978a899b609b6215cb2d474 in 934ms, sequenceid=420, compaction requested=true 2024-11-15T17:40:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-15T17:40:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-15T17:40:08,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T17:40:08,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6340 sec 2024-11-15T17:40:08,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.6410 sec 2024-11-15T17:40:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:08,997 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:08,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T17:40:09,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/05ee994dac9d4feb81a6fea8d7f1e2cc is 50, key is test_row_0/A:col10/1731692408997/Put/seqid=0 2024-11-15T17:40:09,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741927_1103 (size=17181) 2024-11-15T17:40:09,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692469015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692469015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692469017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692469126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692469126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692469126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692469336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692469336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692469336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/05ee994dac9d4feb81a6fea8d7f1e2cc 2024-11-15T17:40:09,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8b069edf0aec4d78b159fea6f5e3ef5f is 50, key is test_row_0/B:col10/1731692408997/Put/seqid=0 2024-11-15T17:40:09,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741928_1104 (size=12301) 2024-11-15T17:40:09,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8b069edf0aec4d78b159fea6f5e3ef5f 2024-11-15T17:40:09,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/f8b232b9a93a4e0590dc185a4ea2be66 is 50, key is test_row_0/C:col10/1731692408997/Put/seqid=0 2024-11-15T17:40:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741929_1105 (size=12301) 2024-11-15T17:40:09,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54466 deadline: 1731692469645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54424 deadline: 1731692469646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54418 deadline: 1731692469646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:09,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/f8b232b9a93a4e0590dc185a4ea2be66 2024-11-15T17:40:09,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/05ee994dac9d4feb81a6fea8d7f1e2cc as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/05ee994dac9d4feb81a6fea8d7f1e2cc 2024-11-15T17:40:09,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/05ee994dac9d4feb81a6fea8d7f1e2cc, entries=250, sequenceid=442, filesize=16.8 K 2024-11-15T17:40:09,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8b069edf0aec4d78b159fea6f5e3ef5f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8b069edf0aec4d78b159fea6f5e3ef5f 2024-11-15T17:40:09,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8b069edf0aec4d78b159fea6f5e3ef5f, entries=150, sequenceid=442, filesize=12.0 K 2024-11-15T17:40:09,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/f8b232b9a93a4e0590dc185a4ea2be66 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8b232b9a93a4e0590dc185a4ea2be66 2024-11-15T17:40:09,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8b232b9a93a4e0590dc185a4ea2be66, entries=150, sequenceid=442, filesize=12.0 K 2024-11-15T17:40:09,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ada8f074c978a899b609b6215cb2d474 in 868ms, sequenceid=442, compaction requested=true 2024-11-15T17:40:09,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:09,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:09,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:09,865 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:09,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:09,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:09,865 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:09,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ada8f074c978a899b609b6215cb2d474:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:09,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:09,867 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:09,867 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:09,867 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/A is initiating minor compaction (all files) 2024-11-15T17:40:09,867 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/B is initiating minor compaction (all files) 2024-11-15T17:40:09,867 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/A in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:09,867 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/B in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:09,867 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/abcb2c23641146f1b8443b4d84f0254c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b7ba9dc054ed42d69361439ee719907a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2d2821686357412c9da9af8c17392a5a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8b069edf0aec4d78b159fea6f5e3ef5f] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=48.9 K 2024-11-15T17:40:09,867 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61592f64fa704afcaa660224c24a9cc0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/b5ce5c0fd10a4644bd1bc2be10094884, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3540b0f358aa430b9d536c2966773783, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/05ee994dac9d4feb81a6fea8d7f1e2cc] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=56.1 K 2024-11-15T17:40:09,868 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61592f64fa704afcaa660224c24a9cc0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731692405322 2024-11-15T17:40:09,868 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting abcb2c23641146f1b8443b4d84f0254c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731692405322 2024-11-15T17:40:09,868 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5ce5c0fd10a4644bd1bc2be10094884, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731692406498 2024-11-15T17:40:09,868 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ba9dc054ed42d69361439ee719907a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731692406498 2024-11-15T17:40:09,868 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3540b0f358aa430b9d536c2966773783, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731692406735 2024-11-15T17:40:09,868 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d2821686357412c9da9af8c17392a5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731692406735 2024-11-15T17:40:09,869 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ee994dac9d4feb81a6fea8d7f1e2cc, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731692408665 2024-11-15T17:40:09,869 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b069edf0aec4d78b159fea6f5e3ef5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731692408674 2024-11-15T17:40:09,878 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:09,879 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/9950edc4b8a64beebdfd50081fde5de0 is 50, key is test_row_0/B:col10/1731692408997/Put/seqid=0 2024-11-15T17:40:09,882 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:09,882 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/9ecb35d7a4134e72bb25d4d464be46ea is 50, key is test_row_0/A:col10/1731692408997/Put/seqid=0 2024-11-15T17:40:09,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741930_1106 (size=13357) 2024-11-15T17:40:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741931_1107 (size=13357) 2024-11-15T17:40:09,900 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/9ecb35d7a4134e72bb25d4d464be46ea as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/9ecb35d7a4134e72bb25d4d464be46ea 2024-11-15T17:40:09,906 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/A of ada8f074c978a899b609b6215cb2d474 into 9ecb35d7a4134e72bb25d4d464be46ea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:09,906 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:09,906 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/A, priority=12, startTime=1731692409865; duration=0sec 2024-11-15T17:40:09,906 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:09,906 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:A 2024-11-15T17:40:09,906 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:09,909 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:09,909 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): ada8f074c978a899b609b6215cb2d474/C is initiating minor compaction (all files) 2024-11-15T17:40:09,909 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ada8f074c978a899b609b6215cb2d474/C in TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:09,909 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c0977207f8466a9122b093bc599082, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/bd3e04ab0b774e9fa60d92bf80b2fb54, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7320942571aa4725b78929813b384b64, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8b232b9a93a4e0590dc185a4ea2be66] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp, totalSize=48.9 K 2024-11-15T17:40:09,910 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02c0977207f8466a9122b093bc599082, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731692405322 2024-11-15T17:40:09,910 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd3e04ab0b774e9fa60d92bf80b2fb54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731692406498 2024-11-15T17:40:09,910 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7320942571aa4725b78929813b384b64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731692406735 2024-11-15T17:40:09,910 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b232b9a93a4e0590dc185a4ea2be66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731692408674 2024-11-15T17:40:09,918 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ada8f074c978a899b609b6215cb2d474#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:09,919 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/d80f63ed362a4b84b8c15ee654bd443e is 50, key is test_row_0/C:col10/1731692408997/Put/seqid=0 2024-11-15T17:40:09,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741932_1108 (size=13357) 2024-11-15T17:40:10,094 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T17:40:10,094 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,096 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T17:40:10,096 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,096 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T17:40:10,096 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,097 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T17:40:10,097 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,098 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,098 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,098 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,098 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,099 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,099 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,099 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,099 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:10,159 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:10,159 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T17:40:10,159 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T17:40:10,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:40:10,160 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T17:40:10,160 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,160 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,160 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,160 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,160 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,160 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,160 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,160 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,160 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/9eaa2960c4ce40fb96d1adaf7d730e3a is 50, key is test_row_0/A:col10/1731692410156/Put/seqid=0 2024-11-15T17:40:10,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741933_1109 (size=12301) 2024-11-15T17:40:10,291 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/9950edc4b8a64beebdfd50081fde5de0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/9950edc4b8a64beebdfd50081fde5de0 2024-11-15T17:40:10,299 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/B of ada8f074c978a899b609b6215cb2d474 into 9950edc4b8a64beebdfd50081fde5de0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:10,300 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:10,300 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/B, priority=12, startTime=1731692409865; duration=0sec 2024-11-15T17:40:10,300 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:10,300 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:B 2024-11-15T17:40:10,329 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/d80f63ed362a4b84b8c15ee654bd443e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/d80f63ed362a4b84b8c15ee654bd443e 2024-11-15T17:40:10,334 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ada8f074c978a899b609b6215cb2d474/C of ada8f074c978a899b609b6215cb2d474 into d80f63ed362a4b84b8c15ee654bd443e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:10,335 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:10,335 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., storeName=ada8f074c978a899b609b6215cb2d474/C, priority=12, startTime=1731692409865; duration=0sec 2024-11-15T17:40:10,335 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:10,335 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ada8f074c978a899b609b6215cb2d474:C 2024-11-15T17:40:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T17:40:10,486 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:10,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/9eaa2960c4ce40fb96d1adaf7d730e3a 2024-11-15T17:40:10,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/7fe3f1a2fb174b49a2ad19d41ea74ddf is 50, key is test_row_0/B:col10/1731692410156/Put/seqid=0 2024-11-15T17:40:10,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741934_1110 (size=12301) 2024-11-15T17:40:10,697 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T17:40:10,697 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,698 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,698 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,938 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T17:40:10,938 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:10,938 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,938 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,938 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:40:10,938 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-15T17:40:10,938 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-15T17:40:10,938 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-15T17:40:10,938 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3322 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3236 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1544 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4627 rows 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1565 2024-11-15T17:40:10,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4684 rows 2024-11-15T17:40:10,939 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:40:10,939 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:40:10,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:10,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:10,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:40:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:40:10,951 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692410951"}]},"ts":"1731692410951"} 2024-11-15T17:40:10,953 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:40:10,953 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:40:10,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:40:10,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, UNASSIGN}] 2024-11-15T17:40:10,961 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, UNASSIGN 2024-11-15T17:40:10,963 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ada8f074c978a899b609b6215cb2d474, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:10,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, UNASSIGN because future has completed 2024-11-15T17:40:10,966 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:40:10,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:40:10,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/7fe3f1a2fb174b49a2ad19d41ea74ddf 2024-11-15T17:40:11,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/f8855d322438405d93c3eea2eeccf71d is 50, key is test_row_0/C:col10/1731692410156/Put/seqid=0 2024-11-15T17:40:11,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741935_1111 (size=12301) 2024-11-15T17:40:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:40:11,125 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:11,125 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:40:11,126 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing ada8f074c978a899b609b6215cb2d474, disabling compactions & flushes 2024-11-15T17:40:11,126 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:11,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:40:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:40:11,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/f8855d322438405d93c3eea2eeccf71d 2024-11-15T17:40:11,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/9eaa2960c4ce40fb96d1adaf7d730e3a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/9eaa2960c4ce40fb96d1adaf7d730e3a 2024-11-15T17:40:11,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/9eaa2960c4ce40fb96d1adaf7d730e3a, entries=150, sequenceid=460, filesize=12.0 K 2024-11-15T17:40:11,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/7fe3f1a2fb174b49a2ad19d41ea74ddf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7fe3f1a2fb174b49a2ad19d41ea74ddf 2024-11-15T17:40:11,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7fe3f1a2fb174b49a2ad19d41ea74ddf, entries=150, sequenceid=460, filesize=12.0 K 2024-11-15T17:40:11,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/f8855d322438405d93c3eea2eeccf71d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8855d322438405d93c3eea2eeccf71d 2024-11-15T17:40:11,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8855d322438405d93c3eea2eeccf71d, entries=150, sequenceid=460, filesize=12.0 K 2024-11-15T17:40:11,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=13.42 KB/13740 for ada8f074c978a899b609b6215cb2d474 in 1286ms, sequenceid=460, compaction requested=false 2024-11-15T17:40:11,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ada8f074c978a899b609b6215cb2d474: 2024-11-15T17:40:11,445 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:11,445 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:11,446 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. after waiting 0 ms 2024-11-15T17:40:11,446 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:11,446 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing ada8f074c978a899b609b6215cb2d474 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-15T17:40:11,452 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/6652bfc0f37f45648afb4db238e06ffd is 50, key is test_row_0/A:col10/1731692410696/Put/seqid=0 2024-11-15T17:40:11,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741936_1112 (size=9857) 2024-11-15T17:40:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:40:11,866 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/6652bfc0f37f45648afb4db238e06ffd 2024-11-15T17:40:11,874 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8367f8c1123d47ce935e9e577d2907b7 is 50, key is test_row_0/B:col10/1731692410696/Put/seqid=0 2024-11-15T17:40:11,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741937_1113 (size=9857) 2024-11-15T17:40:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:40:12,279 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8367f8c1123d47ce935e9e577d2907b7 2024-11-15T17:40:12,289 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/42ffae8f81fb43028b1b01875ac7b6ae is 50, key is test_row_0/C:col10/1731692410696/Put/seqid=0 2024-11-15T17:40:12,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741938_1114 (size=9857) 2024-11-15T17:40:12,696 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/42ffae8f81fb43028b1b01875ac7b6ae 2024-11-15T17:40:12,704 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/A/6652bfc0f37f45648afb4db238e06ffd as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/6652bfc0f37f45648afb4db238e06ffd 2024-11-15T17:40:12,709 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/6652bfc0f37f45648afb4db238e06ffd, entries=100, sequenceid=467, filesize=9.6 K 2024-11-15T17:40:12,710 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/B/8367f8c1123d47ce935e9e577d2907b7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8367f8c1123d47ce935e9e577d2907b7 2024-11-15T17:40:12,714 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8367f8c1123d47ce935e9e577d2907b7, entries=100, sequenceid=467, filesize=9.6 K 2024-11-15T17:40:12,715 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/.tmp/C/42ffae8f81fb43028b1b01875ac7b6ae as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/42ffae8f81fb43028b1b01875ac7b6ae 2024-11-15T17:40:12,720 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/42ffae8f81fb43028b1b01875ac7b6ae, entries=100, sequenceid=467, filesize=9.6 K 2024-11-15T17:40:12,721 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for ada8f074c978a899b609b6215cb2d474 in 1275ms, sequenceid=467, compaction requested=true 2024-11-15T17:40:12,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/df99f74e17014ebeb6a46554fe51d0a2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/7706b57d2f4b42cfaea688a7df571bc3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b1934072f35478cbc53c7bdcd79a081, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/898bd54bb88848959fd2f489403b7fc6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e6ebe6b329014218ac0860ec472fb1cb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/514337ae287d4302b56edb30438c4eec, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/513e722ca03c46e8a1612271ad605df3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/a8ff44e4bb184d538ec68e8b2e9a4571, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/39b81b920e1a4c47a4903fd66ee49c65, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5266f13997ee43cfa5973b11a1704d04, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/2a935101bccb4ebb8c6b8ee103c3dd99, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d152b0f7343048a9905836bb6c16a0de, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/70c4437077a74d26b60902eb9e38236a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/56c71642b92c4ba698d4d8898a599ffb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5a59e24ece3945a18720e3ab55549f79, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/47f774ebd89d4e9590b51138fe722936, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/85e02b055f834df0b3a47dc25541e35f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3e9e229bc37348e09559b6ab8e0836e1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/060a0083db1140b9816286e9da2875a0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d348027823be4173926c0caaae6652f9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/115f639d4aeb49749a942b2c80285671, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/4953350560dd4598844a1eab618b1234, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/8f6e9350a093418a9e0cc9f90fdd1a70, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/0255ccf9cdda48eb887ec4d24c07b230, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b18d43897f74d36931c5c3483857e8c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61134c005d6e47b8bbafbb8f0e0056ab, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e9a82b346a8c47ddb713a4be4e2f276d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61592f64fa704afcaa660224c24a9cc0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/b5ce5c0fd10a4644bd1bc2be10094884, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3540b0f358aa430b9d536c2966773783, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/05ee994dac9d4feb81a6fea8d7f1e2cc] to archive 2024-11-15T17:40:12,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:40:12,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/df99f74e17014ebeb6a46554fe51d0a2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/df99f74e17014ebeb6a46554fe51d0a2 2024-11-15T17:40:12,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/7706b57d2f4b42cfaea688a7df571bc3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/7706b57d2f4b42cfaea688a7df571bc3 2024-11-15T17:40:12,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b1934072f35478cbc53c7bdcd79a081 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b1934072f35478cbc53c7bdcd79a081 2024-11-15T17:40:12,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/898bd54bb88848959fd2f489403b7fc6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/898bd54bb88848959fd2f489403b7fc6 2024-11-15T17:40:12,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e6ebe6b329014218ac0860ec472fb1cb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e6ebe6b329014218ac0860ec472fb1cb 2024-11-15T17:40:12,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/514337ae287d4302b56edb30438c4eec to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/514337ae287d4302b56edb30438c4eec 2024-11-15T17:40:12,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/513e722ca03c46e8a1612271ad605df3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/513e722ca03c46e8a1612271ad605df3 2024-11-15T17:40:12,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/a8ff44e4bb184d538ec68e8b2e9a4571 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/a8ff44e4bb184d538ec68e8b2e9a4571 2024-11-15T17:40:12,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/39b81b920e1a4c47a4903fd66ee49c65 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/39b81b920e1a4c47a4903fd66ee49c65 2024-11-15T17:40:12,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5266f13997ee43cfa5973b11a1704d04 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5266f13997ee43cfa5973b11a1704d04 2024-11-15T17:40:12,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/2a935101bccb4ebb8c6b8ee103c3dd99 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/2a935101bccb4ebb8c6b8ee103c3dd99 2024-11-15T17:40:12,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d152b0f7343048a9905836bb6c16a0de to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d152b0f7343048a9905836bb6c16a0de 2024-11-15T17:40:12,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/70c4437077a74d26b60902eb9e38236a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/70c4437077a74d26b60902eb9e38236a 2024-11-15T17:40:12,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/56c71642b92c4ba698d4d8898a599ffb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/56c71642b92c4ba698d4d8898a599ffb 2024-11-15T17:40:12,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5a59e24ece3945a18720e3ab55549f79 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/5a59e24ece3945a18720e3ab55549f79 2024-11-15T17:40:12,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/47f774ebd89d4e9590b51138fe722936 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/47f774ebd89d4e9590b51138fe722936 2024-11-15T17:40:12,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/85e02b055f834df0b3a47dc25541e35f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/85e02b055f834df0b3a47dc25541e35f 2024-11-15T17:40:12,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3e9e229bc37348e09559b6ab8e0836e1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3e9e229bc37348e09559b6ab8e0836e1 2024-11-15T17:40:12,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/060a0083db1140b9816286e9da2875a0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/060a0083db1140b9816286e9da2875a0 2024-11-15T17:40:12,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d348027823be4173926c0caaae6652f9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/d348027823be4173926c0caaae6652f9 2024-11-15T17:40:12,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/115f639d4aeb49749a942b2c80285671 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/115f639d4aeb49749a942b2c80285671 2024-11-15T17:40:12,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/4953350560dd4598844a1eab618b1234 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/4953350560dd4598844a1eab618b1234 2024-11-15T17:40:12,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/8f6e9350a093418a9e0cc9f90fdd1a70 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/8f6e9350a093418a9e0cc9f90fdd1a70 2024-11-15T17:40:12,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/0255ccf9cdda48eb887ec4d24c07b230 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/0255ccf9cdda48eb887ec4d24c07b230 2024-11-15T17:40:12,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b18d43897f74d36931c5c3483857e8c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/1b18d43897f74d36931c5c3483857e8c 2024-11-15T17:40:12,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61134c005d6e47b8bbafbb8f0e0056ab to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61134c005d6e47b8bbafbb8f0e0056ab 2024-11-15T17:40:12,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e9a82b346a8c47ddb713a4be4e2f276d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/e9a82b346a8c47ddb713a4be4e2f276d 2024-11-15T17:40:12,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61592f64fa704afcaa660224c24a9cc0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/61592f64fa704afcaa660224c24a9cc0 2024-11-15T17:40:12,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/b5ce5c0fd10a4644bd1bc2be10094884 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/b5ce5c0fd10a4644bd1bc2be10094884 2024-11-15T17:40:12,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3540b0f358aa430b9d536c2966773783 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/3540b0f358aa430b9d536c2966773783 2024-11-15T17:40:12,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/05ee994dac9d4feb81a6fea8d7f1e2cc to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/05ee994dac9d4feb81a6fea8d7f1e2cc 2024-11-15T17:40:12,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e97f6f0ec23e4048af8eff916de980ec, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/fa459ba46d9844dcbe0567ce3e6a40b4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7e9a825e4f5549b88026a13010b21a9c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2ea167a40821480c8f690e2cc26ff9ce, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1f9d00ec302e4b15b0158ea4d928fdbc, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/aaa7b29e159e4527a5708c41fc0248b4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e4d87e9520234731946f43c5776ecd65, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/435d6374d80c4f929d86a02d651a9279, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/5d645bb88ba84e5c9a9fceae5b42654d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/c83e8d1e8b3a4a2c9690eb1c383b418c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/04b2199c6ddd49f8b9273abd4d7fb847, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/37288ac2e5bc40689dc40f4540eac24d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ef3498c639b04db8b2d46a25d4859e6a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/3f5a4fe682da4ca19899fe4edc7c4462, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/35164e5e48fe4b04ab865f6039837768, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8daf476c1bfa433c8f71ae0c850ad87e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/bd64c66114b249d087b6a74eea17bd2f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/0e07da88c8664f60a9fef789aed56c2a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/4d95968586484d17b13d21d4fb1883c2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/40bda268beac4880b1e4a867d0662de4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/892ade4f6b0a41a7864d25d87b681f64, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1c1139fc34e347d8a03c850741ec385b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/13af6dbd36e34720857fb9d2cd65f911, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b1f238477ce54a8ea7cf3713bd17a9da, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8eccbfa9b2cf464a96bd0ae2172ef522, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ac53afb1a0174a80b5edc071373c8fdf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/abcb2c23641146f1b8443b4d84f0254c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/542e972f1b52484fbb032aba79a5fa64, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b7ba9dc054ed42d69361439ee719907a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2d2821686357412c9da9af8c17392a5a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8b069edf0aec4d78b159fea6f5e3ef5f] to archive 2024-11-15T17:40:12,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:40:12,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e97f6f0ec23e4048af8eff916de980ec to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e97f6f0ec23e4048af8eff916de980ec 2024-11-15T17:40:12,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/fa459ba46d9844dcbe0567ce3e6a40b4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/fa459ba46d9844dcbe0567ce3e6a40b4 2024-11-15T17:40:12,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7e9a825e4f5549b88026a13010b21a9c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7e9a825e4f5549b88026a13010b21a9c 2024-11-15T17:40:12,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2ea167a40821480c8f690e2cc26ff9ce to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2ea167a40821480c8f690e2cc26ff9ce 2024-11-15T17:40:12,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1f9d00ec302e4b15b0158ea4d928fdbc to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1f9d00ec302e4b15b0158ea4d928fdbc 2024-11-15T17:40:12,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/aaa7b29e159e4527a5708c41fc0248b4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/aaa7b29e159e4527a5708c41fc0248b4 2024-11-15T17:40:12,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e4d87e9520234731946f43c5776ecd65 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/e4d87e9520234731946f43c5776ecd65 2024-11-15T17:40:12,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/435d6374d80c4f929d86a02d651a9279 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/435d6374d80c4f929d86a02d651a9279 2024-11-15T17:40:12,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/5d645bb88ba84e5c9a9fceae5b42654d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/5d645bb88ba84e5c9a9fceae5b42654d 2024-11-15T17:40:12,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/c83e8d1e8b3a4a2c9690eb1c383b418c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/c83e8d1e8b3a4a2c9690eb1c383b418c 2024-11-15T17:40:12,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/04b2199c6ddd49f8b9273abd4d7fb847 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/04b2199c6ddd49f8b9273abd4d7fb847 2024-11-15T17:40:12,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/37288ac2e5bc40689dc40f4540eac24d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/37288ac2e5bc40689dc40f4540eac24d 2024-11-15T17:40:12,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ef3498c639b04db8b2d46a25d4859e6a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ef3498c639b04db8b2d46a25d4859e6a 2024-11-15T17:40:12,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/3f5a4fe682da4ca19899fe4edc7c4462 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/3f5a4fe682da4ca19899fe4edc7c4462 2024-11-15T17:40:12,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/35164e5e48fe4b04ab865f6039837768 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/35164e5e48fe4b04ab865f6039837768 2024-11-15T17:40:12,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8daf476c1bfa433c8f71ae0c850ad87e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8daf476c1bfa433c8f71ae0c850ad87e 2024-11-15T17:40:12,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/bd64c66114b249d087b6a74eea17bd2f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/bd64c66114b249d087b6a74eea17bd2f 2024-11-15T17:40:12,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/0e07da88c8664f60a9fef789aed56c2a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/0e07da88c8664f60a9fef789aed56c2a 2024-11-15T17:40:12,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/4d95968586484d17b13d21d4fb1883c2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/4d95968586484d17b13d21d4fb1883c2 2024-11-15T17:40:12,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/40bda268beac4880b1e4a867d0662de4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/40bda268beac4880b1e4a867d0662de4 2024-11-15T17:40:12,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/892ade4f6b0a41a7864d25d87b681f64 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/892ade4f6b0a41a7864d25d87b681f64 2024-11-15T17:40:12,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1c1139fc34e347d8a03c850741ec385b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/1c1139fc34e347d8a03c850741ec385b 2024-11-15T17:40:12,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/13af6dbd36e34720857fb9d2cd65f911 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/13af6dbd36e34720857fb9d2cd65f911 2024-11-15T17:40:12,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b1f238477ce54a8ea7cf3713bd17a9da to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b1f238477ce54a8ea7cf3713bd17a9da 2024-11-15T17:40:12,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8eccbfa9b2cf464a96bd0ae2172ef522 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8eccbfa9b2cf464a96bd0ae2172ef522 2024-11-15T17:40:12,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ac53afb1a0174a80b5edc071373c8fdf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/ac53afb1a0174a80b5edc071373c8fdf 2024-11-15T17:40:12,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/abcb2c23641146f1b8443b4d84f0254c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/abcb2c23641146f1b8443b4d84f0254c 2024-11-15T17:40:12,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/542e972f1b52484fbb032aba79a5fa64 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/542e972f1b52484fbb032aba79a5fa64 2024-11-15T17:40:12,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b7ba9dc054ed42d69361439ee719907a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/b7ba9dc054ed42d69361439ee719907a 2024-11-15T17:40:12,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2d2821686357412c9da9af8c17392a5a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/2d2821686357412c9da9af8c17392a5a 2024-11-15T17:40:12,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8b069edf0aec4d78b159fea6f5e3ef5f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8b069edf0aec4d78b159fea6f5e3ef5f 2024-11-15T17:40:12,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1ea68ad278514677969cbe5cf8974e10, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/aaa0318194cd42b998f0ddd7838f7491, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/b7561729deaa464b9daa684b6e2b2cc0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8013f06723384f5286f07376e862422f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/da78062f6e9342ada1dd6a7e5e117998, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/cd76dd13f8e44675b4529a1d5fb6fb65, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/2357564ca0a64af3beeb47f207b202fb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/4232c04dfc404c2cb13cfa4d1477712a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/85a72e6e40644fc799b3a0e03464ca52, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/997af595ee794c1098deeba3e732551b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/96c63591d921445a9dd9f02b6aa99037, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c6cb35b57d471cbb85a99f35f7cc2e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/80a29eaa71c54873bda54b131b78dc2e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8088884a47c3402a983217b0cabaa374, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/33be76fc0c4f44e18b797a68caeda054, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6cd1860f828040c8b1e022ce2b652bfa, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7de9ca0ee9b14494b41c738f080576e9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/fd1185870d474d55bec05f30f6965338, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c41333927ec34cd08bd569df32326db7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/e1e79ed844e04e31a8ee5e628d96f3bb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6b1c80d3bf6342429e8d17ad6597f96f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c4d1079d55464cda98bd3db409d10f60, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/3c567255313a4b558e993f3856ee8c0b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7bbaa1d5aed84f78b532d9b1dc536633, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/5ba57c0126584335afaf7915b11f596f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1e5dd0568b1d4e27a04d04b238f0fcc0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c0977207f8466a9122b093bc599082, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/687713a17f714bdfa3f95140fb685574, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/bd3e04ab0b774e9fa60d92bf80b2fb54, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7320942571aa4725b78929813b384b64, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8b232b9a93a4e0590dc185a4ea2be66] to archive 2024-11-15T17:40:12,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:40:12,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1ea68ad278514677969cbe5cf8974e10 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1ea68ad278514677969cbe5cf8974e10 2024-11-15T17:40:12,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/aaa0318194cd42b998f0ddd7838f7491 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/aaa0318194cd42b998f0ddd7838f7491 2024-11-15T17:40:12,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/b7561729deaa464b9daa684b6e2b2cc0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/b7561729deaa464b9daa684b6e2b2cc0 2024-11-15T17:40:12,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8013f06723384f5286f07376e862422f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8013f06723384f5286f07376e862422f 2024-11-15T17:40:12,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/da78062f6e9342ada1dd6a7e5e117998 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/da78062f6e9342ada1dd6a7e5e117998 2024-11-15T17:40:12,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/cd76dd13f8e44675b4529a1d5fb6fb65 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/cd76dd13f8e44675b4529a1d5fb6fb65 2024-11-15T17:40:12,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/2357564ca0a64af3beeb47f207b202fb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/2357564ca0a64af3beeb47f207b202fb 2024-11-15T17:40:12,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/4232c04dfc404c2cb13cfa4d1477712a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/4232c04dfc404c2cb13cfa4d1477712a 2024-11-15T17:40:12,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/85a72e6e40644fc799b3a0e03464ca52 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/85a72e6e40644fc799b3a0e03464ca52 2024-11-15T17:40:12,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/997af595ee794c1098deeba3e732551b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/997af595ee794c1098deeba3e732551b 2024-11-15T17:40:12,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/96c63591d921445a9dd9f02b6aa99037 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/96c63591d921445a9dd9f02b6aa99037 2024-11-15T17:40:12,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c6cb35b57d471cbb85a99f35f7cc2e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c6cb35b57d471cbb85a99f35f7cc2e 2024-11-15T17:40:12,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/80a29eaa71c54873bda54b131b78dc2e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/80a29eaa71c54873bda54b131b78dc2e 2024-11-15T17:40:12,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8088884a47c3402a983217b0cabaa374 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/8088884a47c3402a983217b0cabaa374 2024-11-15T17:40:12,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/33be76fc0c4f44e18b797a68caeda054 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/33be76fc0c4f44e18b797a68caeda054 2024-11-15T17:40:12,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6cd1860f828040c8b1e022ce2b652bfa to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6cd1860f828040c8b1e022ce2b652bfa 2024-11-15T17:40:12,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7de9ca0ee9b14494b41c738f080576e9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7de9ca0ee9b14494b41c738f080576e9 2024-11-15T17:40:12,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/fd1185870d474d55bec05f30f6965338 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/fd1185870d474d55bec05f30f6965338 2024-11-15T17:40:12,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c41333927ec34cd08bd569df32326db7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c41333927ec34cd08bd569df32326db7 2024-11-15T17:40:12,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/e1e79ed844e04e31a8ee5e628d96f3bb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/e1e79ed844e04e31a8ee5e628d96f3bb 2024-11-15T17:40:12,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6b1c80d3bf6342429e8d17ad6597f96f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/6b1c80d3bf6342429e8d17ad6597f96f 2024-11-15T17:40:12,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c4d1079d55464cda98bd3db409d10f60 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/c4d1079d55464cda98bd3db409d10f60 2024-11-15T17:40:12,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/3c567255313a4b558e993f3856ee8c0b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/3c567255313a4b558e993f3856ee8c0b 2024-11-15T17:40:12,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7bbaa1d5aed84f78b532d9b1dc536633 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7bbaa1d5aed84f78b532d9b1dc536633 2024-11-15T17:40:12,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/5ba57c0126584335afaf7915b11f596f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/5ba57c0126584335afaf7915b11f596f 2024-11-15T17:40:12,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1e5dd0568b1d4e27a04d04b238f0fcc0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/1e5dd0568b1d4e27a04d04b238f0fcc0 2024-11-15T17:40:12,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c0977207f8466a9122b093bc599082 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/02c0977207f8466a9122b093bc599082 2024-11-15T17:40:12,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/687713a17f714bdfa3f95140fb685574 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/687713a17f714bdfa3f95140fb685574 2024-11-15T17:40:12,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/bd3e04ab0b774e9fa60d92bf80b2fb54 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/bd3e04ab0b774e9fa60d92bf80b2fb54 2024-11-15T17:40:12,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7320942571aa4725b78929813b384b64 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/7320942571aa4725b78929813b384b64 2024-11-15T17:40:12,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8b232b9a93a4e0590dc185a4ea2be66 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8b232b9a93a4e0590dc185a4ea2be66 2024-11-15T17:40:12,893 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/recovered.edits/470.seqid, newMaxSeqId=470, maxSeqId=1 2024-11-15T17:40:12,894 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. 2024-11-15T17:40:12,894 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for ada8f074c978a899b609b6215cb2d474: Waiting for close lock at 1731692411126Running coprocessor pre-close hooks at 1731692411126Disabling compacts and flushes for region at 1731692411126Disabling writes for close at 1731692411446 (+320 ms)Obtaining lock to block concurrent updates at 1731692411446Preparing flush snapshotting stores in ada8f074c978a899b609b6215cb2d474 at 1731692411446Finished memstore snapshotting TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731692411446Flushing stores of TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474. at 1731692411447 (+1 ms)Flushing ada8f074c978a899b609b6215cb2d474/A: creating writer at 1731692411447Flushing ada8f074c978a899b609b6215cb2d474/A: appending metadata at 1731692411452 (+5 ms)Flushing ada8f074c978a899b609b6215cb2d474/A: closing flushed file at 1731692411452Flushing ada8f074c978a899b609b6215cb2d474/B: creating writer at 1731692411871 (+419 ms)Flushing ada8f074c978a899b609b6215cb2d474/B: appending metadata at 1731692411873 (+2 ms)Flushing ada8f074c978a899b609b6215cb2d474/B: closing flushed file at 1731692411873Flushing ada8f074c978a899b609b6215cb2d474/C: creating writer at 1731692412285 (+412 ms)Flushing ada8f074c978a899b609b6215cb2d474/C: appending metadata at 1731692412288 (+3 ms)Flushing ada8f074c978a899b609b6215cb2d474/C: closing flushed file at 1731692412288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ca4b8cd: reopening flushed file at 1731692412703 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@267322c7: reopening flushed file at 1731692412709 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4666c0b: reopening flushed file at 1731692412714 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for ada8f074c978a899b609b6215cb2d474 in 1275ms, sequenceid=467, compaction requested=true at 1731692412721 (+7 ms)Writing region close event to WAL at 1731692412889 (+168 ms)Running coprocessor post-close hooks at 1731692412894 (+5 ms)Closed at 1731692412894 2024-11-15T17:40:12,896 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:12,897 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ada8f074c978a899b609b6215cb2d474, regionState=CLOSED 2024-11-15T17:40:12,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:40:12,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-15T17:40:12,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure ada8f074c978a899b609b6215cb2d474, server=4e3b17656000,45187,1731692383005 in 1.9340 sec 2024-11-15T17:40:12,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-15T17:40:12,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ada8f074c978a899b609b6215cb2d474, UNASSIGN in 1.9420 sec 2024-11-15T17:40:12,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T17:40:12,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9500 sec 2024-11-15T17:40:12,908 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692412908"}]},"ts":"1731692412908"} 2024-11-15T17:40:12,910 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:40:12,910 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:40:12,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9660 sec 2024-11-15T17:40:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:40:13,096 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:13,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:40:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:13,105 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-15T17:40:13,107 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:13,111 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:13,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/recovered.edits] 2024-11-15T17:40:13,118 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/6652bfc0f37f45648afb4db238e06ffd to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/6652bfc0f37f45648afb4db238e06ffd 2024-11-15T17:40:13,119 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/9eaa2960c4ce40fb96d1adaf7d730e3a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/9eaa2960c4ce40fb96d1adaf7d730e3a 2024-11-15T17:40:13,120 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/9ecb35d7a4134e72bb25d4d464be46ea to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/A/9ecb35d7a4134e72bb25d4d464be46ea 2024-11-15T17:40:13,123 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7fe3f1a2fb174b49a2ad19d41ea74ddf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/7fe3f1a2fb174b49a2ad19d41ea74ddf 2024-11-15T17:40:13,124 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8367f8c1123d47ce935e9e577d2907b7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/8367f8c1123d47ce935e9e577d2907b7 2024-11-15T17:40:13,125 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/9950edc4b8a64beebdfd50081fde5de0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/B/9950edc4b8a64beebdfd50081fde5de0 2024-11-15T17:40:13,128 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/42ffae8f81fb43028b1b01875ac7b6ae to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/42ffae8f81fb43028b1b01875ac7b6ae 2024-11-15T17:40:13,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/d80f63ed362a4b84b8c15ee654bd443e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/d80f63ed362a4b84b8c15ee654bd443e 2024-11-15T17:40:13,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8855d322438405d93c3eea2eeccf71d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/C/f8855d322438405d93c3eea2eeccf71d 2024-11-15T17:40:13,133 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/recovered.edits/470.seqid to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474/recovered.edits/470.seqid 2024-11-15T17:40:13,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/ada8f074c978a899b609b6215cb2d474 2024-11-15T17:40:13,134 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:40:13,140 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:13,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T17:40:13,153 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:40:13,187 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:40:13,189 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:13,189 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:40:13,190 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731692413189"}]},"ts":"9223372036854775807"} 2024-11-15T17:40:13,195 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:40:13,195 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ada8f074c978a899b609b6215cb2d474, NAME => 'TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:40:13,195 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:40:13,196 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731692413196"}]},"ts":"9223372036854775807"} 2024-11-15T17:40:13,199 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:40:13,200 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:13,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-11-15T17:40:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-15T17:40:13,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:40:13,216 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:13,227 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1230616815_22 at /127.0.0.1:51400 [Waiting for operation #162] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4e3b17656000:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1795595999_22 at /127.0.0.1:36546 [Waiting for operation #249] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1230616815_22 at /127.0.0.1:36634 [Waiting for operation #228] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;4e3b17656000:45187-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=392 (was 295) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10076 (was 10174) 2024-11-15T17:40:13,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=392, ProcessCount=11, AvailableMemoryMB=10075 2024-11-15T17:40:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:40:13,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:40:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:13,242 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:40:13,242 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:13,242 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-15T17:40:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T17:40:13,244 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:40:13,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741939_1115 (size=959) 2024-11-15T17:40:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T17:40:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T17:40:13,652 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:40:13,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741940_1116 (size=53) 2024-11-15T17:40:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T17:40:14,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:40:14,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 103dbae58ceff9ab6b93f5c0fda56a23, disabling compactions & flushes 2024-11-15T17:40:14,059 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:14,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:14,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. after waiting 0 ms 2024-11-15T17:40:14,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:14,059 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:14,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 103dbae58ceff9ab6b93f5c0fda56a23: Waiting for close lock at 1731692414059Disabling compacts and flushes for region at 1731692414059Disabling writes for close at 1731692414059Writing region close event to WAL at 1731692414059Closed at 1731692414059 2024-11-15T17:40:14,061 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:40:14,061 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731692414061"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692414061"}]},"ts":"1731692414061"} 2024-11-15T17:40:14,063 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:40:14,065 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:40:14,065 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692414065"}]},"ts":"1731692414065"} 2024-11-15T17:40:14,067 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T17:40:14,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, ASSIGN}] 2024-11-15T17:40:14,068 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, ASSIGN 2024-11-15T17:40:14,069 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, ASSIGN; state=OFFLINE, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=false 2024-11-15T17:40:14,220 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:14,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, ASSIGN because future has completed 2024-11-15T17:40:14,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:40:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T17:40:14,379 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:14,380 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:40:14,380 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,380 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:40:14,380 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,380 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,382 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,384 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 103dbae58ceff9ab6b93f5c0fda56a23 columnFamilyName A 2024-11-15T17:40:14,385 DEBUG [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:14,386 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(327): Store=103dbae58ceff9ab6b93f5c0fda56a23/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:14,386 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,388 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 103dbae58ceff9ab6b93f5c0fda56a23 columnFamilyName B 2024-11-15T17:40:14,388 DEBUG [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:14,388 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(327): Store=103dbae58ceff9ab6b93f5c0fda56a23/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:14,388 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,390 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 103dbae58ceff9ab6b93f5c0fda56a23 columnFamilyName C 2024-11-15T17:40:14,390 DEBUG [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:14,390 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(327): Store=103dbae58ceff9ab6b93f5c0fda56a23/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:14,391 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,392 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,392 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,393 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,393 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,394 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:40:14,395 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,397 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:40:14,398 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 103dbae58ceff9ab6b93f5c0fda56a23; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59773809, jitterRate=-0.10930083692073822}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:40:14,398 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:14,399 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 103dbae58ceff9ab6b93f5c0fda56a23: Running coprocessor pre-open hook at 1731692414380Writing region info on filesystem at 1731692414380Initializing all the Stores at 1731692414381 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692414381Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692414382 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692414382Cleaning up temporary data from old regions at 1731692414393 (+11 ms)Running coprocessor post-open hooks at 1731692414398 (+5 ms)Region opened successfully at 1731692414399 (+1 ms) 2024-11-15T17:40:14,400 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., pid=32, masterSystemTime=1731692414376 2024-11-15T17:40:14,403 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:14,403 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:14,404 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=OPEN, openSeqNum=2, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:14,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:40:14,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T17:40:14,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 in 185 msec 2024-11-15T17:40:14,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-15T17:40:14,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, ASSIGN in 343 msec 2024-11-15T17:40:14,416 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:40:14,416 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692414416"}]},"ts":"1731692414416"} 2024-11-15T17:40:14,419 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T17:40:14,420 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:40:14,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-15T17:40:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T17:40:15,385 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:15,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:15,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:15,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:15,388 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:15,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:15,390 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:15,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b05381, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:15,391 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:15,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:15,395 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:40:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 4e3b17656000,41205,1731692382118 2024-11-15T17:40:15,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14bf5f07 2024-11-15T17:40:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:40:15,397 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:40:15,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:40:15,401 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:40:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:15,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741941_1117 (size=995) 2024-11-15T17:40:15,823 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T17:40:15,823 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T17:40:15,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:40:15,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, REOPEN/MOVE}] 2024-11-15T17:40:15,833 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, REOPEN/MOVE 2024-11-15T17:40:15,834 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:15,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, REOPEN/MOVE because future has completed 2024-11-15T17:40:15,836 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:40:15,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:40:15,989 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:15,989 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:40:15,989 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 103dbae58ceff9ab6b93f5c0fda56a23, disabling compactions & flushes 2024-11-15T17:40:15,989 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:15,989 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:15,989 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. after waiting 0 ms 2024-11-15T17:40:15,989 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:15,993 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T17:40:15,994 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:15,994 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 103dbae58ceff9ab6b93f5c0fda56a23: Waiting for close lock at 1731692415989Running coprocessor pre-close hooks at 1731692415989Disabling compacts and flushes for region at 1731692415989Disabling writes for close at 1731692415989Writing region close event to WAL at 1731692415990 (+1 ms)Running coprocessor post-close hooks at 1731692415994 (+4 ms)Closed at 1731692415994 2024-11-15T17:40:15,995 WARN [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 103dbae58ceff9ab6b93f5c0fda56a23 to self. 2024-11-15T17:40:15,998 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:15,998 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=CLOSED 2024-11-15T17:40:16,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:40:16,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-15T17:40:16,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 in 166 msec 2024-11-15T17:40:16,005 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, REOPEN/MOVE; state=CLOSED, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=true 2024-11-15T17:40:16,156 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, REOPEN/MOVE because future has completed 2024-11-15T17:40:16,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:40:16,314 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,315 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:40:16,315 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,315 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:40:16,315 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,315 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,318 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,324 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 103dbae58ceff9ab6b93f5c0fda56a23 columnFamilyName A 2024-11-15T17:40:16,327 DEBUG [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:16,327 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(327): Store=103dbae58ceff9ab6b93f5c0fda56a23/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:16,328 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,329 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 103dbae58ceff9ab6b93f5c0fda56a23 columnFamilyName B 2024-11-15T17:40:16,329 DEBUG [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:16,329 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(327): Store=103dbae58ceff9ab6b93f5c0fda56a23/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:16,329 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,330 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 103dbae58ceff9ab6b93f5c0fda56a23 columnFamilyName C 2024-11-15T17:40:16,330 DEBUG [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:16,330 INFO [StoreOpener-103dbae58ceff9ab6b93f5c0fda56a23-1 {}] regionserver.HStore(327): Store=103dbae58ceff9ab6b93f5c0fda56a23/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:16,330 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,331 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,332 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,333 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,333 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,333 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:40:16,335 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,335 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 103dbae58ceff9ab6b93f5c0fda56a23; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64333246, jitterRate=-0.041359931230545044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:40:16,335 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,336 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 103dbae58ceff9ab6b93f5c0fda56a23: Running coprocessor pre-open hook at 1731692416315Writing region info on filesystem at 1731692416315Initializing all the Stores at 1731692416316 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692416316Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692416317 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692416317Cleaning up temporary data from old regions at 1731692416333 (+16 ms)Running coprocessor post-open hooks at 1731692416335 (+2 ms)Region opened successfully at 1731692416336 (+1 ms) 2024-11-15T17:40:16,337 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., pid=37, masterSystemTime=1731692416310 2024-11-15T17:40:16,339 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,339 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=OPEN, openSeqNum=5, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:40:16,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-15T17:40:16,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 in 184 msec 2024-11-15T17:40:16,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-15T17:40:16,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, REOPEN/MOVE in 511 msec 2024-11-15T17:40:16,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T17:40:16,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-15T17:40:16,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 947 msec 2024-11-15T17:40:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-15T17:40:16,355 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:16,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,357 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75241f0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,358 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604a2ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,360 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9f9fc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,362 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e97bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,364 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51556ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e3c767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,367 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ef037f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,368 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d2b5af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,370 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49931a11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,372 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17529369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,373 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70eaa779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,375 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef444e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,377 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,379 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efac9a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,381 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b16b65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,383 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c27714a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:16,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:16,385 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428e195e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:16,387 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d82d85e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:16,388 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,388 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,388 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,388 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,388 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,389 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:16,389 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,389 DEBUG [Thread-601 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,390 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T17:40:16,390 DEBUG [Thread-601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,391 DEBUG [Thread-603 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,392 DEBUG [Thread-603 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T17:40:16,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,393 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,394 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,398 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,400 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:16,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:16,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:40:16,426 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:16,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:40:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f615c6cdaac741d284bd5e2eacea129f_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692416424/Put/seqid=0 2024-11-15T17:40:16,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741942_1118 (size=14594) 2024-11-15T17:40:16,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692476483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T17:40:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692476487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692476489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692476491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692476493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692476599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692476607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692476608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692476616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692476616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T17:40:16,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:16,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692476816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692476826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692476826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692476836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692476836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:16,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:16,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:16,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:16,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:16,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:16,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f615c6cdaac741d284bd5e2eacea129f_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f615c6cdaac741d284bd5e2eacea129f_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:16,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/fa7a437076504d6a9e2f017e636940db, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:16,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/fa7a437076504d6a9e2f017e636940db is 175, key is test_row_0/A:col10/1731692416424/Put/seqid=0 2024-11-15T17:40:16,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741943_1119 (size=39549) 2024-11-15T17:40:16,940 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/fa7a437076504d6a9e2f017e636940db 2024-11-15T17:40:16,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/fc71ce1aded74cf286a2295cb991d2ea is 50, key is test_row_0/B:col10/1731692416424/Put/seqid=0 2024-11-15T17:40:16,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741944_1120 (size=12001) 2024-11-15T17:40:16,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/fc71ce1aded74cf286a2295cb991d2ea 2024-11-15T17:40:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T17:40:17,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/68fe7fa3a3044a8888918c6c723b4801 is 50, key is test_row_0/C:col10/1731692416424/Put/seqid=0 2024-11-15T17:40:17,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741945_1121 (size=12001) 2024-11-15T17:40:17,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/68fe7fa3a3044a8888918c6c723b4801 2024-11-15T17:40:17,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/fa7a437076504d6a9e2f017e636940db as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/fa7a437076504d6a9e2f017e636940db 2024-11-15T17:40:17,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/fa7a437076504d6a9e2f017e636940db, entries=200, sequenceid=16, filesize=38.6 K 2024-11-15T17:40:17,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/fc71ce1aded74cf286a2295cb991d2ea as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fc71ce1aded74cf286a2295cb991d2ea 2024-11-15T17:40:17,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fc71ce1aded74cf286a2295cb991d2ea, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T17:40:17,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/68fe7fa3a3044a8888918c6c723b4801 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/68fe7fa3a3044a8888918c6c723b4801 2024-11-15T17:40:17,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/68fe7fa3a3044a8888918c6c723b4801, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T17:40:17,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 103dbae58ceff9ab6b93f5c0fda56a23 in 684ms, sequenceid=16, compaction requested=false 2024-11-15T17:40:17,111 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T17:40:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:17,138 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:17,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:40:17,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111566d93633f5d9419bbcc2cea74bab9047_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692417136/Put/seqid=0 2024-11-15T17:40:17,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692477152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692477154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692477159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692477160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692477162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:17,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741946_1122 (size=17034) 2024-11-15T17:40:17,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692477277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692477277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692477277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692477277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692477277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:17,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:17,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:17,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692477486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692477486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692477486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692477489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692477489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T17:40:17,592 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:17,599 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111566d93633f5d9419bbcc2cea74bab9047_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111566d93633f5d9419bbcc2cea74bab9047_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:17,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d540f18193dc4ee68088f17663593567, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:17,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d540f18193dc4ee68088f17663593567 is 175, key is test_row_0/A:col10/1731692417136/Put/seqid=0 2024-11-15T17:40:17,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741947_1123 (size=48139) 2024-11-15T17:40:17,625 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d540f18193dc4ee68088f17663593567 2024-11-15T17:40:17,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/58275dd8034546dab05b3b53ffd66c44 is 50, key is test_row_0/B:col10/1731692417136/Put/seqid=0 2024-11-15T17:40:17,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741948_1124 (size=12001) 2024-11-15T17:40:17,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/58275dd8034546dab05b3b53ffd66c44 2024-11-15T17:40:17,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/86232356a4fe4dc08936740e0c951bd8 is 50, key is test_row_0/C:col10/1731692417136/Put/seqid=0 2024-11-15T17:40:17,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741949_1125 (size=12001) 2024-11-15T17:40:17,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692477796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692477796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692477796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692477806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692477806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:17,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:17,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:18,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:18,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:18,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/86232356a4fe4dc08936740e0c951bd8 2024-11-15T17:40:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d540f18193dc4ee68088f17663593567 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d540f18193dc4ee68088f17663593567 2024-11-15T17:40:18,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d540f18193dc4ee68088f17663593567, entries=250, sequenceid=42, filesize=47.0 K 2024-11-15T17:40:18,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/58275dd8034546dab05b3b53ffd66c44 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/58275dd8034546dab05b3b53ffd66c44 2024-11-15T17:40:18,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/58275dd8034546dab05b3b53ffd66c44, entries=150, sequenceid=42, filesize=11.7 K 2024-11-15T17:40:18,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/86232356a4fe4dc08936740e0c951bd8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/86232356a4fe4dc08936740e0c951bd8 2024-11-15T17:40:18,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/86232356a4fe4dc08936740e0c951bd8, entries=150, sequenceid=42, filesize=11.7 K 2024-11-15T17:40:18,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1074ms, sequenceid=42, compaction requested=false 2024-11-15T17:40:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:18,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T17:40:18,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:18,256 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T17:40:18,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d53dbc365b274c059d84001b1d786c48_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692417156/Put/seqid=0 2024-11-15T17:40:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:18,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:18,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741950_1126 (size=12154) 2024-11-15T17:40:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692478350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692478354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692478352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692478365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692478368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,449 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:40:18,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692478475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692478475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692478476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692478476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692478485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T17:40:18,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692478686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692478686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692478687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692478687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692478696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:18,731 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d53dbc365b274c059d84001b1d786c48_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d53dbc365b274c059d84001b1d786c48_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bc1b4302a1b3430693cf5461cad53343, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bc1b4302a1b3430693cf5461cad53343 is 175, key is test_row_0/A:col10/1731692417156/Put/seqid=0 2024-11-15T17:40:18,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741951_1127 (size=30955) 2024-11-15T17:40:19,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692478996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692478996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692478996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692478996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692479006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,141 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bc1b4302a1b3430693cf5461cad53343 2024-11-15T17:40:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/64b059bd060a47989d2983327d133a54 is 50, key is test_row_0/B:col10/1731692417156/Put/seqid=0 2024-11-15T17:40:19,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741952_1128 (size=12001) 2024-11-15T17:40:19,199 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/64b059bd060a47989d2983327d133a54 2024-11-15T17:40:19,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d8e02ccd226548d38abd6f408ffc67f9 is 50, key is test_row_0/C:col10/1731692417156/Put/seqid=0 2024-11-15T17:40:19,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741953_1129 (size=12001) 2024-11-15T17:40:19,239 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d8e02ccd226548d38abd6f408ffc67f9 2024-11-15T17:40:19,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bc1b4302a1b3430693cf5461cad53343 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc1b4302a1b3430693cf5461cad53343 2024-11-15T17:40:19,265 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc1b4302a1b3430693cf5461cad53343, entries=150, sequenceid=52, filesize=30.2 K 2024-11-15T17:40:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/64b059bd060a47989d2983327d133a54 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/64b059bd060a47989d2983327d133a54 2024-11-15T17:40:19,278 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/64b059bd060a47989d2983327d133a54, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T17:40:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d8e02ccd226548d38abd6f408ffc67f9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d8e02ccd226548d38abd6f408ffc67f9 2024-11-15T17:40:19,313 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d8e02ccd226548d38abd6f408ffc67f9, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T17:40:19,321 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1066ms, sequenceid=52, compaction requested=true 2024-11-15T17:40:19,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T17:40:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T17:40:19,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T17:40:19,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-11-15T17:40:19,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.9390 sec 2024-11-15T17:40:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:19,521 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:19,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T17:40:19,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692479523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692479523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692479525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692479530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692479531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b210e1b04f164c668744c0ba80c4aeda_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692419521/Put/seqid=0 2024-11-15T17:40:19,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741954_1130 (size=21918) 2024-11-15T17:40:19,564 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,573 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b210e1b04f164c668744c0ba80c4aeda_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b210e1b04f164c668744c0ba80c4aeda_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:19,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f5d2fcdde5a44164add5b15f0932abe5, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:19,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f5d2fcdde5a44164add5b15f0932abe5 is 175, key is test_row_0/A:col10/1731692419521/Put/seqid=0 2024-11-15T17:40:19,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741955_1131 (size=65323) 2024-11-15T17:40:19,597 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f5d2fcdde5a44164add5b15f0932abe5 2024-11-15T17:40:19,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/da8024f7ae1047bb9b5f66a0f2205988 is 50, key is test_row_0/B:col10/1731692419521/Put/seqid=0 2024-11-15T17:40:19,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741956_1132 (size=12001) 2024-11-15T17:40:19,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/da8024f7ae1047bb9b5f66a0f2205988 2024-11-15T17:40:19,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692479638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692479638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692479639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692479646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692479646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:19,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/ad654418c44742caa536c8e3bc1f16ed is 50, key is test_row_0/C:col10/1731692419521/Put/seqid=0 2024-11-15T17:40:19,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741957_1133 (size=12001) 2024-11-15T17:40:19,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/ad654418c44742caa536c8e3bc1f16ed 2024-11-15T17:40:19,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f5d2fcdde5a44164add5b15f0932abe5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f5d2fcdde5a44164add5b15f0932abe5 2024-11-15T17:40:19,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f5d2fcdde5a44164add5b15f0932abe5, entries=350, sequenceid=82, filesize=63.8 K 2024-11-15T17:40:19,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/da8024f7ae1047bb9b5f66a0f2205988 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/da8024f7ae1047bb9b5f66a0f2205988 2024-11-15T17:40:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/da8024f7ae1047bb9b5f66a0f2205988, entries=150, sequenceid=82, filesize=11.7 K 2024-11-15T17:40:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/ad654418c44742caa536c8e3bc1f16ed as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/ad654418c44742caa536c8e3bc1f16ed 2024-11-15T17:40:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/ad654418c44742caa536c8e3bc1f16ed, entries=150, sequenceid=82, filesize=11.7 K 2024-11-15T17:40:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 103dbae58ceff9ab6b93f5c0fda56a23 in 182ms, sequenceid=82, compaction requested=true 2024-11-15T17:40:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:19,703 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:19,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T17:40:19,703 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,705 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:19,705 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:19,705 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:19,705 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fc71ce1aded74cf286a2295cb991d2ea, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/58275dd8034546dab05b3b53ffd66c44, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/64b059bd060a47989d2983327d133a54, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/da8024f7ae1047bb9b5f66a0f2205988] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=46.9 K 2024-11-15T17:40:19,706 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc71ce1aded74cf286a2295cb991d2ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692416423 2024-11-15T17:40:19,706 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 183966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:19,706 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:19,706 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:19,706 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/fa7a437076504d6a9e2f017e636940db, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d540f18193dc4ee68088f17663593567, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc1b4302a1b3430693cf5461cad53343, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f5d2fcdde5a44164add5b15f0932abe5] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=179.7 K 2024-11-15T17:40:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,706 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:19,707 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/fa7a437076504d6a9e2f017e636940db, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d540f18193dc4ee68088f17663593567, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc1b4302a1b3430693cf5461cad53343, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f5d2fcdde5a44164add5b15f0932abe5] 2024-11-15T17:40:19,707 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58275dd8034546dab05b3b53ffd66c44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731692416489 2024-11-15T17:40:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,707 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7a437076504d6a9e2f017e636940db, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692416420 2024-11-15T17:40:19,708 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64b059bd060a47989d2983327d133a54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692417156 2024-11-15T17:40:19,709 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting da8024f7ae1047bb9b5f66a0f2205988, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731692419518 2024-11-15T17:40:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,711 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d540f18193dc4ee68088f17663593567, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731692416466 2024-11-15T17:40:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,712 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1b4302a1b3430693cf5461cad53343, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692417156 2024-11-15T17:40:19,712 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d2fcdde5a44164add5b15f0932abe5, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731692418349 2024-11-15T17:40:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,739 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,749 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#114 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:19,750 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/716ebe1fbd7f40b6aaf1dc81520d40c5 is 50, key is test_row_0/B:col10/1731692419521/Put/seqid=0 2024-11-15T17:40:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,763 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411159ba36f0bf4ee42e186370b21759c89be_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741958_1134 (size=12139) 2024-11-15T17:40:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,776 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/716ebe1fbd7f40b6aaf1dc81520d40c5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/716ebe1fbd7f40b6aaf1dc81520d40c5 2024-11-15T17:40:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,781 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411159ba36f0bf4ee42e186370b21759c89be_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:19,782 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159ba36f0bf4ee42e186370b21759c89be_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,788 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into 716ebe1fbd7f40b6aaf1dc81520d40c5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:19,788 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:19,788 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=12, startTime=1731692419703; duration=0sec 2024-11-15T17:40:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,788 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:19,788 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:19,788 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,791 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:19,791 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/C is initiating minor compaction (all files) 2024-11-15T17:40:19,791 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/C in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:19,791 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/68fe7fa3a3044a8888918c6c723b4801, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/86232356a4fe4dc08936740e0c951bd8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d8e02ccd226548d38abd6f408ffc67f9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/ad654418c44742caa536c8e3bc1f16ed] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=46.9 K 2024-11-15T17:40:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,792 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68fe7fa3a3044a8888918c6c723b4801, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692416423 2024-11-15T17:40:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,793 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86232356a4fe4dc08936740e0c951bd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731692416489 2024-11-15T17:40:19,793 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e02ccd226548d38abd6f408ffc67f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692417156 2024-11-15T17:40:19,794 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad654418c44742caa536c8e3bc1f16ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731692419518 2024-11-15T17:40:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741959_1135 (size=4469) 2024-11-15T17:40:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,814 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#115 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:19,814 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bc62cb491a38441a915b9acd5e7f7511 is 175, key is test_row_0/A:col10/1731692419521/Put/seqid=0 2024-11-15T17:40:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,830 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#C#compaction#116 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,830 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/709be03a64aa45f3b77aa356e261ed30 is 50, key is test_row_0/C:col10/1731692419521/Put/seqid=0 2024-11-15T17:40:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741961_1137 (size=12139) 2024-11-15T17:40:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741960_1136 (size=31093) 2024-11-15T17:40:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,852 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/709be03a64aa45f3b77aa356e261ed30 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/709be03a64aa45f3b77aa356e261ed30 2024-11-15T17:40:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,860 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/C of 103dbae58ceff9ab6b93f5c0fda56a23 into 709be03a64aa45f3b77aa356e261ed30(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:19,860 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:19,860 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/C, priority=12, startTime=1731692419703; duration=0sec 2024-11-15T17:40:19,860 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:19,860 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:19,888 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:19,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:40:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:19,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111540c8e28f3ab5424fbfa8c72886f1e2ba_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692419888/Put/seqid=0 2024-11-15T17:40:19,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741962_1138 (size=14594) 2024-11-15T17:40:19,939 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:19,945 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111540c8e28f3ab5424fbfa8c72886f1e2ba_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111540c8e28f3ab5424fbfa8c72886f1e2ba_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:19,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/c101e7370eec4e4980131f65e66dfeba, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:19,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/c101e7370eec4e4980131f65e66dfeba is 175, key is test_row_0/A:col10/1731692419888/Put/seqid=0 2024-11-15T17:40:19,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741963_1139 (size=39549) 2024-11-15T17:40:19,957 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/c101e7370eec4e4980131f65e66dfeba 2024-11-15T17:40:19,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c87431cfe9de400c9225555f108eaec3 is 50, key is test_row_0/B:col10/1731692419888/Put/seqid=0 2024-11-15T17:40:19,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741964_1140 (size=12001) 2024-11-15T17:40:19,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c87431cfe9de400c9225555f108eaec3 2024-11-15T17:40:19,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d767cfa4115b4316ab05ba97c8854028 is 50, key is test_row_0/C:col10/1731692419888/Put/seqid=0 2024-11-15T17:40:20,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741965_1141 (size=12001) 2024-11-15T17:40:20,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d767cfa4115b4316ab05ba97c8854028 2024-11-15T17:40:20,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/c101e7370eec4e4980131f65e66dfeba as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/c101e7370eec4e4980131f65e66dfeba 2024-11-15T17:40:20,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/c101e7370eec4e4980131f65e66dfeba, entries=200, sequenceid=96, filesize=38.6 K 2024-11-15T17:40:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c87431cfe9de400c9225555f108eaec3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c87431cfe9de400c9225555f108eaec3 2024-11-15T17:40:20,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c87431cfe9de400c9225555f108eaec3, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T17:40:20,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d767cfa4115b4316ab05ba97c8854028 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d767cfa4115b4316ab05ba97c8854028 2024-11-15T17:40:20,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d767cfa4115b4316ab05ba97c8854028, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T17:40:20,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=127.47 KB/130530 for 103dbae58ceff9ab6b93f5c0fda56a23 in 142ms, sequenceid=96, compaction requested=false 2024-11-15T17:40:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:20,039 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:20,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:40:20,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115daefbb70ab7f4f3abfbbedbcef04ae73_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692420032/Put/seqid=0 2024-11-15T17:40:20,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741966_1142 (size=17034) 2024-11-15T17:40:20,083 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:20,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115daefbb70ab7f4f3abfbbedbcef04ae73_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115daefbb70ab7f4f3abfbbedbcef04ae73_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:20,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/3db5f30fddef4e6ba2dab30f6dc1e42b, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:20,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/3db5f30fddef4e6ba2dab30f6dc1e42b is 175, key is test_row_0/A:col10/1731692420032/Put/seqid=0 2024-11-15T17:40:20,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692480085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692480093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692480099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692480107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692480107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741967_1143 (size=48139) 2024-11-15T17:40:20,145 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/3db5f30fddef4e6ba2dab30f6dc1e42b 2024-11-15T17:40:20,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/31e31320203e4d688497f359971fc50d is 50, key is test_row_0/B:col10/1731692420032/Put/seqid=0 2024-11-15T17:40:20,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741968_1144 (size=12001) 2024-11-15T17:40:20,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692480216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692480216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692480216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692480216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692480217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,256 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bc62cb491a38441a915b9acd5e7f7511 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc62cb491a38441a915b9acd5e7f7511 2024-11-15T17:40:20,267 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into bc62cb491a38441a915b9acd5e7f7511(size=30.4 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:20,267 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:20,268 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=12, startTime=1731692419703; duration=0sec 2024-11-15T17:40:20,268 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:20,268 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692480425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692480426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692480426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692480436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692480436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T17:40:20,545 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:20,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T17:40:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T17:40:20,548 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:20,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:20,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T17:40:20,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/31e31320203e4d688497f359971fc50d 2024-11-15T17:40:20,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/05a3329c2cef4e0baab1232750f46eb7 is 50, key is test_row_0/C:col10/1731692420032/Put/seqid=0 2024-11-15T17:40:20,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T17:40:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:20,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:20,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741969_1145 (size=12001) 2024-11-15T17:40:20,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/05a3329c2cef4e0baab1232750f46eb7 2024-11-15T17:40:20,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/3db5f30fddef4e6ba2dab30f6dc1e42b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3db5f30fddef4e6ba2dab30f6dc1e42b 2024-11-15T17:40:20,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3db5f30fddef4e6ba2dab30f6dc1e42b, entries=250, sequenceid=120, filesize=47.0 K 2024-11-15T17:40:20,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/31e31320203e4d688497f359971fc50d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/31e31320203e4d688497f359971fc50d 2024-11-15T17:40:20,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692480746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692480746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692480746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692480748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692480748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:20,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/31e31320203e4d688497f359971fc50d, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T17:40:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/05a3329c2cef4e0baab1232750f46eb7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/05a3329c2cef4e0baab1232750f46eb7 2024-11-15T17:40:20,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/05a3329c2cef4e0baab1232750f46eb7, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T17:40:20,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 103dbae58ceff9ab6b93f5c0fda56a23 in 739ms, sequenceid=120, compaction requested=true 2024-11-15T17:40:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:20,778 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:20,780 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118781 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:20,780 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:20,780 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:20,780 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc62cb491a38441a915b9acd5e7f7511, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/c101e7370eec4e4980131f65e66dfeba, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3db5f30fddef4e6ba2dab30f6dc1e42b] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=116.0 K 2024-11-15T17:40:20,780 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:20,780 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc62cb491a38441a915b9acd5e7f7511, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/c101e7370eec4e4980131f65e66dfeba, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3db5f30fddef4e6ba2dab30f6dc1e42b] 2024-11-15T17:40:20,783 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bc62cb491a38441a915b9acd5e7f7511, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731692419518 2024-11-15T17:40:20,783 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c101e7370eec4e4980131f65e66dfeba, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731692419872 2024-11-15T17:40:20,784 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db5f30fddef4e6ba2dab30f6dc1e42b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731692419972 2024-11-15T17:40:20,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:20,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:20,792 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:20,793 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:20,793 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:20,793 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:20,793 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/716ebe1fbd7f40b6aaf1dc81520d40c5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c87431cfe9de400c9225555f108eaec3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/31e31320203e4d688497f359971fc50d] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=35.3 K 2024-11-15T17:40:20,794 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 716ebe1fbd7f40b6aaf1dc81520d40c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731692419518 2024-11-15T17:40:20,794 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c87431cfe9de400c9225555f108eaec3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731692419872 2024-11-15T17:40:20,795 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31e31320203e4d688497f359971fc50d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731692419972 2024-11-15T17:40:20,799 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:20,817 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:20,818 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b028ead211794d23b06801dfea8d8c12_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:20,819 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/f2c739ea0a25484a8cecf19655f431ac is 50, key is test_row_0/B:col10/1731692420032/Put/seqid=0 2024-11-15T17:40:20,821 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b028ead211794d23b06801dfea8d8c12_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:20,821 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b028ead211794d23b06801dfea8d8c12_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:20,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T17:40:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:20,858 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:40:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T17:40:20,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741971_1147 (size=4469) 2024-11-15T17:40:20,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741970_1146 (size=12241) 2024-11-15T17:40:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157b10131348db4308bbf6bb10e4b6c628_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692420095/Put/seqid=0 2024-11-15T17:40:20,904 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/f2c739ea0a25484a8cecf19655f431ac as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f2c739ea0a25484a8cecf19655f431ac 2024-11-15T17:40:20,913 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into f2c739ea0a25484a8cecf19655f431ac(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:20,913 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:20,913 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=13, startTime=1731692420792; duration=0sec 2024-11-15T17:40:20,913 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:20,913 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:20,913 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:20,915 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:20,915 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/C is initiating minor compaction (all files) 2024-11-15T17:40:20,915 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/C in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:20,915 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/709be03a64aa45f3b77aa356e261ed30, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d767cfa4115b4316ab05ba97c8854028, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/05a3329c2cef4e0baab1232750f46eb7] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=35.3 K 2024-11-15T17:40:20,916 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709be03a64aa45f3b77aa356e261ed30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731692419518 2024-11-15T17:40:20,916 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d767cfa4115b4316ab05ba97c8854028, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731692419872 2024-11-15T17:40:20,917 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a3329c2cef4e0baab1232750f46eb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731692419972 2024-11-15T17:40:20,934 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:20,935 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4c905e9263584eac9aef62be5a5b3f89 is 50, key is test_row_0/C:col10/1731692420032/Put/seqid=0 2024-11-15T17:40:20,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741972_1148 (size=12304) 2024-11-15T17:40:20,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741973_1149 (size=12241) 2024-11-15T17:40:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T17:40:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:21,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:21,271 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#123 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:21,271 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f4090f19e7f341c398430a787d6ce995 is 175, key is test_row_0/A:col10/1731692420032/Put/seqid=0 2024-11-15T17:40:21,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741974_1150 (size=31195) 2024-11-15T17:40:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692481294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692481296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692481298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692481300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692481308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,319 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f4090f19e7f341c398430a787d6ce995 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f4090f19e7f341c398430a787d6ce995 2024-11-15T17:40:21,340 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into f4090f19e7f341c398430a787d6ce995(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:21,340 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:21,340 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=13, startTime=1731692420778; duration=0sec 2024-11-15T17:40:21,340 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:21,340 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:21,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:21,364 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157b10131348db4308bbf6bb10e4b6c628_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157b10131348db4308bbf6bb10e4b6c628_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ecc0878b7b0a4cddbd7b54c1a50ebed4, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ecc0878b7b0a4cddbd7b54c1a50ebed4 is 175, key is test_row_0/A:col10/1731692420095/Put/seqid=0 2024-11-15T17:40:21,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741975_1151 (size=31105) 2024-11-15T17:40:21,378 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ecc0878b7b0a4cddbd7b54c1a50ebed4 2024-11-15T17:40:21,378 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4c905e9263584eac9aef62be5a5b3f89 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4c905e9263584eac9aef62be5a5b3f89 2024-11-15T17:40:21,392 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/C of 103dbae58ceff9ab6b93f5c0fda56a23 into 4c905e9263584eac9aef62be5a5b3f89(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:21,392 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:21,392 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/C, priority=13, startTime=1731692420800; duration=0sec 2024-11-15T17:40:21,392 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:21,392 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:21,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/dd299ab3c4414869860fc768784acddf is 50, key is test_row_0/B:col10/1731692420095/Put/seqid=0 2024-11-15T17:40:21,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741976_1152 (size=12151) 2024-11-15T17:40:21,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692481415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692481416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692481416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692481416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692481416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692481626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692481626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692481626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692481626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692481627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T17:40:21,806 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/dd299ab3c4414869860fc768784acddf 2024-11-15T17:40:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/49be2912ebb142a78a34f0fc83b8704e is 50, key is test_row_0/C:col10/1731692420095/Put/seqid=0 2024-11-15T17:40:21,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741977_1153 (size=12151) 2024-11-15T17:40:21,849 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/49be2912ebb142a78a34f0fc83b8704e 2024-11-15T17:40:21,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ecc0878b7b0a4cddbd7b54c1a50ebed4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ecc0878b7b0a4cddbd7b54c1a50ebed4 2024-11-15T17:40:21,875 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ecc0878b7b0a4cddbd7b54c1a50ebed4, entries=150, sequenceid=134, filesize=30.4 K 2024-11-15T17:40:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/dd299ab3c4414869860fc768784acddf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/dd299ab3c4414869860fc768784acddf 2024-11-15T17:40:21,890 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/dd299ab3c4414869860fc768784acddf, entries=150, sequenceid=134, filesize=11.9 K 2024-11-15T17:40:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/49be2912ebb142a78a34f0fc83b8704e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/49be2912ebb142a78a34f0fc83b8704e 2024-11-15T17:40:21,899 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/49be2912ebb142a78a34f0fc83b8704e, entries=150, sequenceid=134, filesize=11.9 K 2024-11-15T17:40:21,901 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1043ms, sequenceid=134, compaction requested=false 2024-11-15T17:40:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T17:40:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T17:40:21,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T17:40:21,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3540 sec 2024-11-15T17:40:21,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3600 sec 2024-11-15T17:40:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:21,936 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:21,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:40:21,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155311bd80fa5a473ca9b6fb8bebc4c22c_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692421935/Put/seqid=0 2024-11-15T17:40:21,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741978_1154 (size=17284) 2024-11-15T17:40:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692481951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,958 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692481951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692481952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692481956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155311bd80fa5a473ca9b6fb8bebc4c22c_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155311bd80fa5a473ca9b6fb8bebc4c22c_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:21,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bf52cf93fcd844ee9f9f15b09c4a2365, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:21,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bf52cf93fcd844ee9f9f15b09c4a2365 is 175, key is test_row_0/A:col10/1731692421935/Put/seqid=0 2024-11-15T17:40:21,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692481961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:21,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741979_1155 (size=48389) 2024-11-15T17:40:21,989 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bf52cf93fcd844ee9f9f15b09c4a2365 2024-11-15T17:40:22,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/790e250f9ba44ee29b7037f9c5164f07 is 50, key is test_row_0/B:col10/1731692421935/Put/seqid=0 2024-11-15T17:40:22,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741980_1156 (size=12151) 2024-11-15T17:40:22,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/790e250f9ba44ee29b7037f9c5164f07 2024-11-15T17:40:22,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/362b042b0c894aaa8668ec7cbaf00904 is 50, key is test_row_0/C:col10/1731692421935/Put/seqid=0 2024-11-15T17:40:22,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741981_1157 (size=12151) 2024-11-15T17:40:22,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692482066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692482066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692482066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692482066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692482077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692482276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692482286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692482286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692482286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692482286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/362b042b0c894aaa8668ec7cbaf00904 2024-11-15T17:40:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bf52cf93fcd844ee9f9f15b09c4a2365 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bf52cf93fcd844ee9f9f15b09c4a2365 2024-11-15T17:40:22,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bf52cf93fcd844ee9f9f15b09c4a2365, entries=250, sequenceid=162, filesize=47.3 K 2024-11-15T17:40:22,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/790e250f9ba44ee29b7037f9c5164f07 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/790e250f9ba44ee29b7037f9c5164f07 2024-11-15T17:40:22,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/790e250f9ba44ee29b7037f9c5164f07, entries=150, sequenceid=162, filesize=11.9 K 2024-11-15T17:40:22,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/362b042b0c894aaa8668ec7cbaf00904 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/362b042b0c894aaa8668ec7cbaf00904 2024-11-15T17:40:22,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/362b042b0c894aaa8668ec7cbaf00904, entries=150, sequenceid=162, filesize=11.9 K 2024-11-15T17:40:22,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 103dbae58ceff9ab6b93f5c0fda56a23 in 569ms, sequenceid=162, compaction requested=true 2024-11-15T17:40:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:22,507 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:22,507 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:22,509 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:22,510 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:22,510 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:22,510 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f2c739ea0a25484a8cecf19655f431ac, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/dd299ab3c4414869860fc768784acddf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/790e250f9ba44ee29b7037f9c5164f07] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=35.7 K 2024-11-15T17:40:22,510 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:22,510 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:22,510 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:22,511 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f4090f19e7f341c398430a787d6ce995, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ecc0878b7b0a4cddbd7b54c1a50ebed4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bf52cf93fcd844ee9f9f15b09c4a2365] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=108.1 K 2024-11-15T17:40:22,511 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:22,511 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f4090f19e7f341c398430a787d6ce995, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ecc0878b7b0a4cddbd7b54c1a50ebed4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bf52cf93fcd844ee9f9f15b09c4a2365] 2024-11-15T17:40:22,511 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c739ea0a25484a8cecf19655f431ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731692419972 2024-11-15T17:40:22,512 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4090f19e7f341c398430a787d6ce995, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731692419972 2024-11-15T17:40:22,512 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting dd299ab3c4414869860fc768784acddf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692420090 2024-11-15T17:40:22,513 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc0878b7b0a4cddbd7b54c1a50ebed4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692420090 2024-11-15T17:40:22,513 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 790e250f9ba44ee29b7037f9c5164f07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731692421298 2024-11-15T17:40:22,514 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf52cf93fcd844ee9f9f15b09c4a2365, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731692421289 2024-11-15T17:40:22,527 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:22,532 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:22,532 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/b473a56c86a54c02a986d8e55c43f2d6 is 50, key is test_row_0/B:col10/1731692421935/Put/seqid=0 2024-11-15T17:40:22,539 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111569e1e51400b647ed8b132c36ce39e7e4_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:22,542 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111569e1e51400b647ed8b132c36ce39e7e4_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:22,542 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569e1e51400b647ed8b132c36ce39e7e4_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:22,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741982_1158 (size=12493) 2024-11-15T17:40:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:22,587 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:22,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:40:22,593 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/b473a56c86a54c02a986d8e55c43f2d6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b473a56c86a54c02a986d8e55c43f2d6 2024-11-15T17:40:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741983_1159 (size=4469) 2024-11-15T17:40:22,602 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#132 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:22,602 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/3e3211c9749f4068b73eb80b26de94b3 is 175, key is test_row_0/A:col10/1731692421935/Put/seqid=0 2024-11-15T17:40:22,604 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into b473a56c86a54c02a986d8e55c43f2d6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:22,604 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:22,605 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=13, startTime=1731692422507; duration=0sec 2024-11-15T17:40:22,605 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:22,605 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:22,605 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:22,607 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:22,607 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/C is initiating minor compaction (all files) 2024-11-15T17:40:22,607 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/C in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:22,607 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4c905e9263584eac9aef62be5a5b3f89, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/49be2912ebb142a78a34f0fc83b8704e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/362b042b0c894aaa8668ec7cbaf00904] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=35.7 K 2024-11-15T17:40:22,607 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c905e9263584eac9aef62be5a5b3f89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731692419972 2024-11-15T17:40:22,608 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 49be2912ebb142a78a34f0fc83b8704e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692420090 2024-11-15T17:40:22,608 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 362b042b0c894aaa8668ec7cbaf00904, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731692421298 2024-11-15T17:40:22,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115849cb1ea2373434aade7405ec0d49bdb_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692421952/Put/seqid=0 2024-11-15T17:40:22,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741984_1160 (size=31447) 2024-11-15T17:40:22,634 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/3e3211c9749f4068b73eb80b26de94b3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3e3211c9749f4068b73eb80b26de94b3 2024-11-15T17:40:22,637 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:22,637 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/1d87d78f2b3e4ae0b3bca8b067a16507 is 50, key is test_row_0/C:col10/1731692421935/Put/seqid=0 2024-11-15T17:40:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692482628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692482629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,645 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into 3e3211c9749f4068b73eb80b26de94b3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:22,645 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:22,645 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=13, startTime=1731692422507; duration=0sec 2024-11-15T17:40:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,645 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:22,645 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692482634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692482634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692482640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741985_1161 (size=14794) 2024-11-15T17:40:22,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:22,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741986_1162 (size=12493) 2024-11-15T17:40:22,669 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115849cb1ea2373434aade7405ec0d49bdb_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115849cb1ea2373434aade7405ec0d49bdb_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:22,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/4dce059031e54d8e811f2fcc9bcc6fed, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:22,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/4dce059031e54d8e811f2fcc9bcc6fed is 175, key is test_row_0/A:col10/1731692421952/Put/seqid=0 2024-11-15T17:40:22,678 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/1d87d78f2b3e4ae0b3bca8b067a16507 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/1d87d78f2b3e4ae0b3bca8b067a16507 2024-11-15T17:40:22,689 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/C of 103dbae58ceff9ab6b93f5c0fda56a23 into 1d87d78f2b3e4ae0b3bca8b067a16507(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:22,689 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:22,689 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/C, priority=13, startTime=1731692422507; duration=0sec 2024-11-15T17:40:22,689 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:22,689 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T17:40:22,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741987_1163 (size=39749) 2024-11-15T17:40:22,700 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:22,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/4dce059031e54d8e811f2fcc9bcc6fed 2024-11-15T17:40:22,700 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T17:40:22,702 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T17:40:22,705 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:22,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:22,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/5225e171f3794a848b1d6a24aab1f087 is 50, key is test_row_0/B:col10/1731692421952/Put/seqid=0 2024-11-15T17:40:22,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741988_1164 (size=12151) 2024-11-15T17:40:22,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/5225e171f3794a848b1d6a24aab1f087 2024-11-15T17:40:22,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4828b869d63147bfb184d175606c01da is 50, key is test_row_0/C:col10/1731692421952/Put/seqid=0 2024-11-15T17:40:22,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692482746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692482748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692482756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692482756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692482756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741989_1165 (size=12151) 2024-11-15T17:40:22,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4828b869d63147bfb184d175606c01da 2024-11-15T17:40:22,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/4dce059031e54d8e811f2fcc9bcc6fed as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/4dce059031e54d8e811f2fcc9bcc6fed 2024-11-15T17:40:22,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/4dce059031e54d8e811f2fcc9bcc6fed, entries=200, sequenceid=175, filesize=38.8 K 2024-11-15T17:40:22,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/5225e171f3794a848b1d6a24aab1f087 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/5225e171f3794a848b1d6a24aab1f087 2024-11-15T17:40:22,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/5225e171f3794a848b1d6a24aab1f087, entries=150, sequenceid=175, filesize=11.9 K 2024-11-15T17:40:22,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4828b869d63147bfb184d175606c01da as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4828b869d63147bfb184d175606c01da 2024-11-15T17:40:22,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4828b869d63147bfb184d175606c01da, entries=150, sequenceid=175, filesize=11.9 K 2024-11-15T17:40:22,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 103dbae58ceff9ab6b93f5c0fda56a23 in 212ms, sequenceid=175, compaction requested=false 2024-11-15T17:40:22,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T17:40:22,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T17:40:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:22,858 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:40:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155d6ad2bcc07c49b6abea06342a093bea_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692422623/Put/seqid=0 2024-11-15T17:40:22,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741990_1166 (size=12304) 2024-11-15T17:40:22,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:22,889 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155d6ad2bcc07c49b6abea06342a093bea_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155d6ad2bcc07c49b6abea06342a093bea_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/03c36bc04c4446aabe0e731a25542017, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/03c36bc04c4446aabe0e731a25542017 is 175, key is test_row_0/A:col10/1731692422623/Put/seqid=0 2024-11-15T17:40:22,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741991_1167 (size=31105) 2024-11-15T17:40:22,921 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/03c36bc04c4446aabe0e731a25542017 2024-11-15T17:40:22,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c12543d9ff1b4092ab37f2fcc191f489 is 50, key is test_row_0/B:col10/1731692422623/Put/seqid=0 2024-11-15T17:40:22,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:22,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741992_1168 (size=12151) 2024-11-15T17:40:22,972 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c12543d9ff1b4092ab37f2fcc191f489 2024-11-15T17:40:22,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692482976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692482977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692482977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692482978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692482980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:22,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/203d3fd420cc423f9dafce95b9d1ad00 is 50, key is test_row_0/C:col10/1731692422623/Put/seqid=0 2024-11-15T17:40:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T17:40:23,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741993_1169 (size=12151) 2024-11-15T17:40:23,033 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/203d3fd420cc423f9dafce95b9d1ad00 2024-11-15T17:40:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/03c36bc04c4446aabe0e731a25542017 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/03c36bc04c4446aabe0e731a25542017 2024-11-15T17:40:23,049 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/03c36bc04c4446aabe0e731a25542017, entries=150, sequenceid=201, filesize=30.4 K 2024-11-15T17:40:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c12543d9ff1b4092ab37f2fcc191f489 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c12543d9ff1b4092ab37f2fcc191f489 2024-11-15T17:40:23,054 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c12543d9ff1b4092ab37f2fcc191f489, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T17:40:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/203d3fd420cc423f9dafce95b9d1ad00 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/203d3fd420cc423f9dafce95b9d1ad00 2024-11-15T17:40:23,067 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/203d3fd420cc423f9dafce95b9d1ad00, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T17:40:23,068 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 103dbae58ceff9ab6b93f5c0fda56a23 in 210ms, sequenceid=201, compaction requested=true 2024-11-15T17:40:23,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:23,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T17:40:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T17:40:23,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T17:40:23,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 366 msec 2024-11-15T17:40:23,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 374 msec 2024-11-15T17:40:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:23,098 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:23,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:40:23,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b16caad8d40c4ff5839d316b8b623c11_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692423098/Put/seqid=0 2024-11-15T17:40:23,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692483122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692483122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692483123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692483131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692483135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741994_1170 (size=14794) 2024-11-15T17:40:23,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692483235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692483246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692483246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692483246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692483246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T17:40:23,335 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:23,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T17:40:23,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T17:40:23,339 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:23,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T17:40:23,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692483445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692483456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692483457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692483457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692483457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:23,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:23,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:23,568 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b16caad8d40c4ff5839d316b8b623c11_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b16caad8d40c4ff5839d316b8b623c11_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:23,572 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/cd8b7305e6fe41eeb7c6ac40c284879b, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:23,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/cd8b7305e6fe41eeb7c6ac40c284879b is 175, key is test_row_0/A:col10/1731692423098/Put/seqid=0 2024-11-15T17:40:23,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741995_1171 (size=39749) 2024-11-15T17:40:23,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:23,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:23,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T17:40:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692483755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692483765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692483766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692483766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692483767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:23,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:23,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:23,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:23,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:23,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:23,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T17:40:24,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/cd8b7305e6fe41eeb7c6ac40c284879b 2024-11-15T17:40:24,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/d07c7d260d7941fbb31b3764d432de5e is 50, key is test_row_0/B:col10/1731692423098/Put/seqid=0 2024-11-15T17:40:24,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741996_1172 (size=12151) 2024-11-15T17:40:24,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/d07c7d260d7941fbb31b3764d432de5e 2024-11-15T17:40:24,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d4a0dfbd85794fd597f9886065896ffd is 50, key is test_row_0/C:col10/1731692423098/Put/seqid=0 2024-11-15T17:40:24,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741997_1173 (size=12151) 2024-11-15T17:40:24,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692484265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692484275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692484275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692484276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:24,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692484286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:24,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:24,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d4a0dfbd85794fd597f9886065896ffd 2024-11-15T17:40:24,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/cd8b7305e6fe41eeb7c6ac40c284879b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/cd8b7305e6fe41eeb7c6ac40c284879b 2024-11-15T17:40:24,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/cd8b7305e6fe41eeb7c6ac40c284879b, entries=200, sequenceid=217, filesize=38.8 K 2024-11-15T17:40:24,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/d07c7d260d7941fbb31b3764d432de5e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d07c7d260d7941fbb31b3764d432de5e 2024-11-15T17:40:24,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d07c7d260d7941fbb31b3764d432de5e, entries=150, sequenceid=217, filesize=11.9 K 2024-11-15T17:40:24,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d4a0dfbd85794fd597f9886065896ffd as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d4a0dfbd85794fd597f9886065896ffd 2024-11-15T17:40:24,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d4a0dfbd85794fd597f9886065896ffd, entries=150, sequenceid=217, filesize=11.9 K 2024-11-15T17:40:24,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1374ms, sequenceid=217, compaction requested=true 2024-11-15T17:40:24,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:24,473 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:24,473 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:24,475 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:24,475 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:24,475 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,475 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3e3211c9749f4068b73eb80b26de94b3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/4dce059031e54d8e811f2fcc9bcc6fed, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/03c36bc04c4446aabe0e731a25542017, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/cd8b7305e6fe41eeb7c6ac40c284879b] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=138.7 K 2024-11-15T17:40:24,475 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,475 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3e3211c9749f4068b73eb80b26de94b3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/4dce059031e54d8e811f2fcc9bcc6fed, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/03c36bc04c4446aabe0e731a25542017, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/cd8b7305e6fe41eeb7c6ac40c284879b] 2024-11-15T17:40:24,475 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:24,476 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:24,476 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,476 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b473a56c86a54c02a986d8e55c43f2d6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/5225e171f3794a848b1d6a24aab1f087, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c12543d9ff1b4092ab37f2fcc191f489, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d07c7d260d7941fbb31b3764d432de5e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=47.8 K 2024-11-15T17:40:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T17:40:24,479 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b473a56c86a54c02a986d8e55c43f2d6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731692421298 2024-11-15T17:40:24,479 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3211c9749f4068b73eb80b26de94b3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731692421298 2024-11-15T17:40:24,480 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dce059031e54d8e811f2fcc9bcc6fed, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731692421943 2024-11-15T17:40:24,480 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 5225e171f3794a848b1d6a24aab1f087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731692421943 2024-11-15T17:40:24,480 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c36bc04c4446aabe0e731a25542017, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731692422623 2024-11-15T17:40:24,480 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c12543d9ff1b4092ab37f2fcc191f489, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731692422623 2024-11-15T17:40:24,481 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd8b7305e6fe41eeb7c6ac40c284879b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731692422963 2024-11-15T17:40:24,481 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d07c7d260d7941fbb31b3764d432de5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731692422963 2024-11-15T17:40:24,499 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:24,500 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/fe043b5afc314a77bc4d41831dbf2123 is 50, key is test_row_0/B:col10/1731692423098/Put/seqid=0 2024-11-15T17:40:24,502 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:24,524 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115333e4f3eb94147eab4a121ae45a0c0ac_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:24,527 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115333e4f3eb94147eab4a121ae45a0c0ac_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:24,527 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115333e4f3eb94147eab4a121ae45a0c0ac_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741998_1174 (size=12629) 2024-11-15T17:40:24,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741999_1175 (size=4469) 2024-11-15T17:40:24,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T17:40:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,578 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:40:24,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c9bceaeb4ef3421090bc03d46f7f4234_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692423120/Put/seqid=0 2024-11-15T17:40:24,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742000_1176 (size=12304) 2024-11-15T17:40:24,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,634 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c9bceaeb4ef3421090bc03d46f7f4234_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c9bceaeb4ef3421090bc03d46f7f4234_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ae7c5f7e115a4dddafb56593369fdcd2, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ae7c5f7e115a4dddafb56593369fdcd2 is 175, key is test_row_0/A:col10/1731692423120/Put/seqid=0 2024-11-15T17:40:24,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742001_1177 (size=31105) 2024-11-15T17:40:24,671 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ae7c5f7e115a4dddafb56593369fdcd2 2024-11-15T17:40:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/d1ee2c791b2845c0a96e566e4c29c982 is 50, key is test_row_0/B:col10/1731692423120/Put/seqid=0 2024-11-15T17:40:24,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742002_1178 (size=12151) 2024-11-15T17:40:24,690 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/d1ee2c791b2845c0a96e566e4c29c982 2024-11-15T17:40:24,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/be7af70a70f84f09a3fc97bd14749a2f is 50, key is test_row_0/C:col10/1731692423120/Put/seqid=0 2024-11-15T17:40:24,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742003_1179 (size=12151) 2024-11-15T17:40:24,725 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/be7af70a70f84f09a3fc97bd14749a2f 2024-11-15T17:40:24,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ae7c5f7e115a4dddafb56593369fdcd2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ae7c5f7e115a4dddafb56593369fdcd2 2024-11-15T17:40:24,752 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ae7c5f7e115a4dddafb56593369fdcd2, entries=150, sequenceid=237, filesize=30.4 K 2024-11-15T17:40:24,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/d1ee2c791b2845c0a96e566e4c29c982 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d1ee2c791b2845c0a96e566e4c29c982 2024-11-15T17:40:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,758 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d1ee2c791b2845c0a96e566e4c29c982, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T17:40:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/be7af70a70f84f09a3fc97bd14749a2f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/be7af70a70f84f09a3fc97bd14749a2f 2024-11-15T17:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,769 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/be7af70a70f84f09a3fc97bd14749a2f, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T17:40:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,772 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 103dbae58ceff9ab6b93f5c0fda56a23 in 193ms, sequenceid=237, compaction requested=true 2024-11-15T17:40:24,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:24,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T17:40:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T17:40:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T17:40:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-15T17:40:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4420 sec 2024-11-15T17:40:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,950 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/fe043b5afc314a77bc4d41831dbf2123 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fe043b5afc314a77bc4d41831dbf2123 2024-11-15T17:40:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,957 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into fe043b5afc314a77bc4d41831dbf2123(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:24,957 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:24,957 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=12, startTime=1731692424473; duration=0sec 2024-11-15T17:40:24,957 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:24,957 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:24,957 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T17:40:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,960 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61097 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T17:40:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,960 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/C is initiating minor compaction (all files) 2024-11-15T17:40:24,960 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/C in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:24,960 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/1d87d78f2b3e4ae0b3bca8b067a16507, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4828b869d63147bfb184d175606c01da, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/203d3fd420cc423f9dafce95b9d1ad00, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d4a0dfbd85794fd597f9886065896ffd, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/be7af70a70f84f09a3fc97bd14749a2f] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=59.7 K 2024-11-15T17:40:24,961 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d87d78f2b3e4ae0b3bca8b067a16507, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731692421298 2024-11-15T17:40:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,962 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#145 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:24,962 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4828b869d63147bfb184d175606c01da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731692421943 2024-11-15T17:40:24,963 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/27c7b7899c964a66a4a9c55fd7191d37 is 175, key is test_row_0/A:col10/1731692423098/Put/seqid=0 2024-11-15T17:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,963 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 203d3fd420cc423f9dafce95b9d1ad00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731692422623 2024-11-15T17:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,964 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a0dfbd85794fd597f9886065896ffd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731692422963 2024-11-15T17:40:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,964 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting be7af70a70f84f09a3fc97bd14749a2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731692423120 2024-11-15T17:40:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742004_1180 (size=31583) 2024-11-15T17:40:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,984 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/27c7b7899c964a66a4a9c55fd7191d37 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/27c7b7899c964a66a4a9c55fd7191d37 2024-11-15T17:40:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,992 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#C#compaction#149 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,993 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/8b4479ca54ea4103ba738f58049ceb74 is 50, key is test_row_0/C:col10/1731692423120/Put/seqid=0 2024-11-15T17:40:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,997 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into 27c7b7899c964a66a4a9c55fd7191d37(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:24,997 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:24,997 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=12, startTime=1731692424473; duration=0sec 2024-11-15T17:40:24,997 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:24,998 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742005_1181 (size=12663) 2024-11-15T17:40:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,062 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/8b4479ca54ea4103ba738f58049ceb74 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/8b4479ca54ea4103ba738f58049ceb74 2024-11-15T17:40:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,069 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/C of 103dbae58ceff9ab6b93f5c0fda56a23 into 8b4479ca54ea4103ba738f58049ceb74(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:25,069 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:25,069 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/C, priority=11, startTime=1731692424473; duration=0sec 2024-11-15T17:40:25,069 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:25,069 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:25,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:40:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:25,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115153dd6623276445a918fadcab165e497_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692425385/Put/seqid=0 2024-11-15T17:40:25,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692485436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692485439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692485442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692485440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692485443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742006_1182 (size=14794) 2024-11-15T17:40:25,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115153dd6623276445a918fadcab165e497_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115153dd6623276445a918fadcab165e497_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:25,467 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/46fb930fb6fd4bd0aa650a37326b00ab, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:25,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/46fb930fb6fd4bd0aa650a37326b00ab is 175, key is test_row_0/A:col10/1731692425385/Put/seqid=0 2024-11-15T17:40:25,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742007_1183 (size=39749) 2024-11-15T17:40:25,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/46fb930fb6fd4bd0aa650a37326b00ab 2024-11-15T17:40:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T17:40:25,485 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:25,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T17:40:25,489 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T17:40:25,489 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:25,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:25,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/f31a85b7bbd3486e82c531ffc6b28265 is 50, key is test_row_0/B:col10/1731692425385/Put/seqid=0 2024-11-15T17:40:25,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742008_1184 (size=12151) 2024-11-15T17:40:25,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/f31a85b7bbd3486e82c531ffc6b28265 2024-11-15T17:40:25,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692485547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692485547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/a34b3eea54bb4fa2ae47918b9044c364 is 50, key is test_row_0/C:col10/1731692425385/Put/seqid=0 2024-11-15T17:40:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692485556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692485556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692485556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742009_1185 (size=12151) 2024-11-15T17:40:25,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/a34b3eea54bb4fa2ae47918b9044c364 2024-11-15T17:40:25,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/46fb930fb6fd4bd0aa650a37326b00ab as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/46fb930fb6fd4bd0aa650a37326b00ab 2024-11-15T17:40:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T17:40:25,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/46fb930fb6fd4bd0aa650a37326b00ab, entries=200, sequenceid=253, filesize=38.8 K 2024-11-15T17:40:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/f31a85b7bbd3486e82c531ffc6b28265 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f31a85b7bbd3486e82c531ffc6b28265 2024-11-15T17:40:25,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f31a85b7bbd3486e82c531ffc6b28265, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T17:40:25,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/a34b3eea54bb4fa2ae47918b9044c364 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/a34b3eea54bb4fa2ae47918b9044c364 2024-11-15T17:40:25,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/a34b3eea54bb4fa2ae47918b9044c364, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T17:40:25,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 103dbae58ceff9ab6b93f5c0fda56a23 in 222ms, sequenceid=253, compaction requested=true 2024-11-15T17:40:25,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:25,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:25,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:25,623 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:25,623 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:25,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:25,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:25,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:25,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:25,625 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:25,625 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:25,625 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:25,626 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fe043b5afc314a77bc4d41831dbf2123, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d1ee2c791b2845c0a96e566e4c29c982, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f31a85b7bbd3486e82c531ffc6b28265] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=36.1 K 2024-11-15T17:40:25,626 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:25,626 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:25,626 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:25,626 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/27c7b7899c964a66a4a9c55fd7191d37, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ae7c5f7e115a4dddafb56593369fdcd2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/46fb930fb6fd4bd0aa650a37326b00ab] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=100.0 K 2024-11-15T17:40:25,626 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:25,626 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/27c7b7899c964a66a4a9c55fd7191d37, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ae7c5f7e115a4dddafb56593369fdcd2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/46fb930fb6fd4bd0aa650a37326b00ab] 2024-11-15T17:40:25,627 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fe043b5afc314a77bc4d41831dbf2123, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731692422963 2024-11-15T17:40:25,627 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ee2c791b2845c0a96e566e4c29c982, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731692423120 2024-11-15T17:40:25,627 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c7b7899c964a66a4a9c55fd7191d37, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731692422963 2024-11-15T17:40:25,627 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f31a85b7bbd3486e82c531ffc6b28265, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731692425385 2024-11-15T17:40:25,627 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae7c5f7e115a4dddafb56593369fdcd2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731692423120 2024-11-15T17:40:25,628 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46fb930fb6fd4bd0aa650a37326b00ab, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731692425385 2024-11-15T17:40:25,637 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:25,638 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/6ddfeafd623f4786aa45b4d79d4a4f6e is 50, key is test_row_0/B:col10/1731692425385/Put/seqid=0 2024-11-15T17:40:25,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T17:40:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:25,643 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:40:25,647 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:25,653 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111590157a8568924b14a142a2d7834c9dd8_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:25,655 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111590157a8568924b14a142a2d7834c9dd8_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:25,656 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111590157a8568924b14a142a2d7834c9dd8_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111529696087daf54b8a9c0fea63ccdaf513_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692425440/Put/seqid=0 2024-11-15T17:40:25,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742010_1186 (size=12731) 2024-11-15T17:40:25,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742011_1187 (size=4469) 2024-11-15T17:40:25,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742012_1188 (size=12454) 2024-11-15T17:40:25,677 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#154 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:25,678 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/43931d4fac4249efb4505c01ed100542 is 175, key is test_row_0/A:col10/1731692425385/Put/seqid=0 2024-11-15T17:40:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:25,700 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111529696087daf54b8a9c0fea63ccdaf513_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111529696087daf54b8a9c0fea63ccdaf513_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:25,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bd1a26e3dd334def90281a832a1a4891, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bd1a26e3dd334def90281a832a1a4891 is 175, key is test_row_0/A:col10/1731692425440/Put/seqid=0 2024-11-15T17:40:25,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742013_1189 (size=31685) 2024-11-15T17:40:25,722 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/43931d4fac4249efb4505c01ed100542 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/43931d4fac4249efb4505c01ed100542 2024-11-15T17:40:25,731 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into 43931d4fac4249efb4505c01ed100542(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:25,731 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:25,731 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=13, startTime=1731692425623; duration=0sec 2024-11-15T17:40:25,731 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:25,731 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:25,732 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T17:40:25,732 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:40:25,732 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:40:25,733 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. because compaction request was cancelled 2024-11-15T17:40:25,733 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:25,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742014_1190 (size=31255) 2024-11-15T17:40:25,743 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bd1a26e3dd334def90281a832a1a4891 2024-11-15T17:40:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/72f7c7f8dffe423c908cb1072ce5622e is 50, key is test_row_0/B:col10/1731692425440/Put/seqid=0 2024-11-15T17:40:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:25,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:25,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742015_1191 (size=12301) 2024-11-15T17:40:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T17:40:25,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692485779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692485777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692485828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692485834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692485828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692485935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692485946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692485946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692485946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692485946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,070 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/6ddfeafd623f4786aa45b4d79d4a4f6e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/6ddfeafd623f4786aa45b4d79d4a4f6e 2024-11-15T17:40:26,075 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into 6ddfeafd623f4786aa45b4d79d4a4f6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:26,075 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:26,075 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=13, startTime=1731692425623; duration=0sec 2024-11-15T17:40:26,075 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:26,075 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T17:40:26,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692486146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692486156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692486156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692486156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692486156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,193 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/72f7c7f8dffe423c908cb1072ce5622e 2024-11-15T17:40:26,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4276c0d2a8544318807127e1a4c2b312 is 50, key is test_row_0/C:col10/1731692425440/Put/seqid=0 2024-11-15T17:40:26,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742016_1192 (size=12301) 2024-11-15T17:40:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692486456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692486465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692486465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692486465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692486465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:26,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T17:40:26,629 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4276c0d2a8544318807127e1a4c2b312 2024-11-15T17:40:26,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bd1a26e3dd334def90281a832a1a4891 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bd1a26e3dd334def90281a832a1a4891 2024-11-15T17:40:26,637 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bd1a26e3dd334def90281a832a1a4891, entries=150, sequenceid=276, filesize=30.5 K 2024-11-15T17:40:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/72f7c7f8dffe423c908cb1072ce5622e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/72f7c7f8dffe423c908cb1072ce5622e 2024-11-15T17:40:26,642 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/72f7c7f8dffe423c908cb1072ce5622e, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T17:40:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/4276c0d2a8544318807127e1a4c2b312 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4276c0d2a8544318807127e1a4c2b312 2024-11-15T17:40:26,648 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4276c0d2a8544318807127e1a4c2b312, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T17:40:26,649 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1005ms, sequenceid=276, compaction requested=true 2024-11-15T17:40:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T17:40:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T17:40:26,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T17:40:26,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1600 sec 2024-11-15T17:40:26,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1660 sec 2024-11-15T17:40:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:26,969 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:26,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T17:40:26,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b9385ce0024c4a5586ed20581d4a1bb0_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692425792/Put/seqid=0 2024-11-15T17:40:26,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742017_1193 (size=12454) 2024-11-15T17:40:26,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:27,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b9385ce0024c4a5586ed20581d4a1bb0_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b9385ce0024c4a5586ed20581d4a1bb0_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:27,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f97ea72ee7d54ccdafc8bcbfbf20deb2, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:27,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f97ea72ee7d54ccdafc8bcbfbf20deb2 is 175, key is test_row_0/A:col10/1731692425792/Put/seqid=0 2024-11-15T17:40:27,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692487003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692487006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692487007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692487008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692487010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742018_1194 (size=31255) 2024-11-15T17:40:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692487116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692487126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692487126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692487126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692487126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692487326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692487336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692487336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692487336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692487336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f97ea72ee7d54ccdafc8bcbfbf20deb2 2024-11-15T17:40:27,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/4d43930a894342ddb73c5023ddb52be5 is 50, key is test_row_0/B:col10/1731692425792/Put/seqid=0 2024-11-15T17:40:27,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742019_1195 (size=12301) 2024-11-15T17:40:27,600 INFO [master/4e3b17656000:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T17:40:27,600 INFO [master/4e3b17656000:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T17:40:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T17:40:27,636 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:27,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692487636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T17:40:27,638 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T17:40:27,638 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:27,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:27,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692487645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692487645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692487645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692487655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T17:40:27,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T17:40:27,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:27,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:27,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:27,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:27,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:27,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/4d43930a894342ddb73c5023ddb52be5 2024-11-15T17:40:27,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/c2378223f7f34df3ab49b6e1224838c2 is 50, key is test_row_0/C:col10/1731692425792/Put/seqid=0 2024-11-15T17:40:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742020_1196 (size=12301) 2024-11-15T17:40:27,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T17:40:27,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:27,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:27,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:27,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:27,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T17:40:28,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T17:40:28,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:28,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:28,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:28,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692488146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:28,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:28,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692488155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692488155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692488156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:28,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692488166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:28,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T17:40:28,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:28,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:28,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T17:40:28,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/c2378223f7f34df3ab49b6e1224838c2 2024-11-15T17:40:28,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/f97ea72ee7d54ccdafc8bcbfbf20deb2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f97ea72ee7d54ccdafc8bcbfbf20deb2 2024-11-15T17:40:28,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f97ea72ee7d54ccdafc8bcbfbf20deb2, entries=150, sequenceid=293, filesize=30.5 K 2024-11-15T17:40:28,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/4d43930a894342ddb73c5023ddb52be5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4d43930a894342ddb73c5023ddb52be5 2024-11-15T17:40:28,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4d43930a894342ddb73c5023ddb52be5, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T17:40:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/c2378223f7f34df3ab49b6e1224838c2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/c2378223f7f34df3ab49b6e1224838c2 2024-11-15T17:40:28,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/c2378223f7f34df3ab49b6e1224838c2, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T17:40:28,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1361ms, sequenceid=293, compaction requested=true 2024-11-15T17:40:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:28,330 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:28,330 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:28,333 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:28,333 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:28,333 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:28,333 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:28,333 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,333 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,333 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/43931d4fac4249efb4505c01ed100542, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bd1a26e3dd334def90281a832a1a4891, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f97ea72ee7d54ccdafc8bcbfbf20deb2] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=92.0 K 2024-11-15T17:40:28,333 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/6ddfeafd623f4786aa45b4d79d4a4f6e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/72f7c7f8dffe423c908cb1072ce5622e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4d43930a894342ddb73c5023ddb52be5] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=36.5 K 2024-11-15T17:40:28,333 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,333 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/43931d4fac4249efb4505c01ed100542, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bd1a26e3dd334def90281a832a1a4891, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f97ea72ee7d54ccdafc8bcbfbf20deb2] 2024-11-15T17:40:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:28,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:28,334 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ddfeafd623f4786aa45b4d79d4a4f6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731692425385 2024-11-15T17:40:28,334 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43931d4fac4249efb4505c01ed100542, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731692425385 2024-11-15T17:40:28,334 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f7c7f8dffe423c908cb1072ce5622e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731692425438 2024-11-15T17:40:28,335 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd1a26e3dd334def90281a832a1a4891, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731692425438 2024-11-15T17:40:28,335 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d43930a894342ddb73c5023ddb52be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731692425771 2024-11-15T17:40:28,335 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f97ea72ee7d54ccdafc8bcbfbf20deb2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731692425771 2024-11-15T17:40:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:28,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:28,347 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:28,358 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:28,359 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c54703d276864aefadb06e76dfb9d601 is 50, key is test_row_0/B:col10/1731692425792/Put/seqid=0 2024-11-15T17:40:28,360 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111580374fed21bd4727aa60b22b8886736b_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:28,362 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111580374fed21bd4727aa60b22b8886736b_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:28,362 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111580374fed21bd4727aa60b22b8886736b_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:28,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742021_1197 (size=4469) 2024-11-15T17:40:28,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742022_1198 (size=12983) 2024-11-15T17:40:28,392 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#161 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:28,392 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/0bed777d379c43118661a5358589377e is 175, key is test_row_0/A:col10/1731692425792/Put/seqid=0 2024-11-15T17:40:28,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T17:40:28,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,404 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T17:40:28,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742023_1199 (size=31937) 2024-11-15T17:40:28,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111554bae4e2d0dd4c69b8a081abeae8edc6_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692427005/Put/seqid=0 2024-11-15T17:40:28,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742024_1200 (size=12454) 2024-11-15T17:40:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T17:40:28,796 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/c54703d276864aefadb06e76dfb9d601 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c54703d276864aefadb06e76dfb9d601 2024-11-15T17:40:28,805 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into c54703d276864aefadb06e76dfb9d601(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:28,805 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:28,805 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=13, startTime=1731692428330; duration=0sec 2024-11-15T17:40:28,806 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:28,806 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:28,806 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:28,814 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/0bed777d379c43118661a5358589377e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0bed777d379c43118661a5358589377e 2024-11-15T17:40:28,815 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:28,818 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/C is initiating minor compaction (all files) 2024-11-15T17:40:28,818 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/C in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:28,818 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/8b4479ca54ea4103ba738f58049ceb74, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/a34b3eea54bb4fa2ae47918b9044c364, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4276c0d2a8544318807127e1a4c2b312, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/c2378223f7f34df3ab49b6e1224838c2] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=48.3 K 2024-11-15T17:40:28,823 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into 0bed777d379c43118661a5358589377e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:28,823 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:28,823 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=13, startTime=1731692428330; duration=0sec 2024-11-15T17:40:28,823 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4479ca54ea4103ba738f58049ceb74, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731692423120 2024-11-15T17:40:28,825 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting a34b3eea54bb4fa2ae47918b9044c364, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731692425385 2024-11-15T17:40:28,825 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:28,826 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:28,826 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4276c0d2a8544318807127e1a4c2b312, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731692425438 2024-11-15T17:40:28,827 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c2378223f7f34df3ab49b6e1224838c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731692425771 2024-11-15T17:40:28,849 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:28,849 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/38bc57d763ce422d98f14de6fe84037f is 50, key is test_row_0/C:col10/1731692425792/Put/seqid=0 2024-11-15T17:40:28,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:28,861 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111554bae4e2d0dd4c69b8a081abeae8edc6_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111554bae4e2d0dd4c69b8a081abeae8edc6_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/73788571b29a4a3d9384548b95313820, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:28,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/73788571b29a4a3d9384548b95313820 is 175, key is test_row_0/A:col10/1731692427005/Put/seqid=0 2024-11-15T17:40:28,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742025_1201 (size=12949) 2024-11-15T17:40:28,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742026_1202 (size=31255) 2024-11-15T17:40:28,919 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/73788571b29a4a3d9384548b95313820 2024-11-15T17:40:28,920 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/38bc57d763ce422d98f14de6fe84037f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/38bc57d763ce422d98f14de6fe84037f 2024-11-15T17:40:28,929 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/C of 103dbae58ceff9ab6b93f5c0fda56a23 into 38bc57d763ce422d98f14de6fe84037f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:28,929 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:28,929 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/C, priority=12, startTime=1731692428335; duration=0sec 2024-11-15T17:40:28,929 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:28,929 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:28,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/4cacb1c63e79477aa990ba98fe1bb00e is 50, key is test_row_0/B:col10/1731692427005/Put/seqid=0 2024-11-15T17:40:28,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742027_1203 (size=12301) 2024-11-15T17:40:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:29,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692489185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692489186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692489193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692489193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692489188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692489296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692489306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692489306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692489307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692489315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,358 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/4cacb1c63e79477aa990ba98fe1bb00e 2024-11-15T17:40:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/53b180082bd04082850279e19148cb43 is 50, key is test_row_0/C:col10/1731692427005/Put/seqid=0 2024-11-15T17:40:29,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742028_1204 (size=12301) 2024-11-15T17:40:29,416 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/53b180082bd04082850279e19148cb43 2024-11-15T17:40:29,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/73788571b29a4a3d9384548b95313820 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/73788571b29a4a3d9384548b95313820 2024-11-15T17:40:29,447 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/73788571b29a4a3d9384548b95313820, entries=150, sequenceid=314, filesize=30.5 K 2024-11-15T17:40:29,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/4cacb1c63e79477aa990ba98fe1bb00e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4cacb1c63e79477aa990ba98fe1bb00e 2024-11-15T17:40:29,456 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4cacb1c63e79477aa990ba98fe1bb00e, entries=150, sequenceid=314, filesize=12.0 K 2024-11-15T17:40:29,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/53b180082bd04082850279e19148cb43 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/53b180082bd04082850279e19148cb43 2024-11-15T17:40:29,464 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/53b180082bd04082850279e19148cb43, entries=150, sequenceid=314, filesize=12.0 K 2024-11-15T17:40:29,466 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1061ms, sequenceid=314, compaction requested=false 2024-11-15T17:40:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T17:40:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T17:40:29,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T17:40:29,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8300 sec 2024-11-15T17:40:29,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8370 sec 2024-11-15T17:40:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:29,507 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:29,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:40:29,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111527bce343f2c74c71ac3622b4d5e9f185_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692429505/Put/seqid=0 2024-11-15T17:40:29,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742029_1205 (size=12454) 2024-11-15T17:40:29,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:29,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692489544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692489548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692489549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,569 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111527bce343f2c74c71ac3622b4d5e9f185_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111527bce343f2c74c71ac3622b4d5e9f185_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692489559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,571 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/8a9e9d2b10b3449480da3f393ac19e1b, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:29,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/8a9e9d2b10b3449480da3f393ac19e1b is 175, key is test_row_0/A:col10/1731692429505/Put/seqid=0 2024-11-15T17:40:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692489567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742030_1206 (size=31255) 2024-11-15T17:40:29,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692489665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692489665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692489666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692489676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692489687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T17:40:29,786 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:29,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T17:40:29,790 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T17:40:29,792 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:29,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:29,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692489875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692489876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692489885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692489896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692489896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T17:40:29,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T17:40:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:29,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,013 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/8a9e9d2b10b3449480da3f393ac19e1b 2024-11-15T17:40:30,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/bd9a91e5df39428690379202d4bf88e2 is 50, key is test_row_0/B:col10/1731692429505/Put/seqid=0 2024-11-15T17:40:30,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742031_1207 (size=12301) 2024-11-15T17:40:30,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/bd9a91e5df39428690379202d4bf88e2 2024-11-15T17:40:30,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d5e8e8b63c914caea6095cc1a28a54e2 is 50, key is test_row_0/C:col10/1731692429505/Put/seqid=0 2024-11-15T17:40:30,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T17:40:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T17:40:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742032_1208 (size=12301) 2024-11-15T17:40:30,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d5e8e8b63c914caea6095cc1a28a54e2 2024-11-15T17:40:30,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/8a9e9d2b10b3449480da3f393ac19e1b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/8a9e9d2b10b3449480da3f393ac19e1b 2024-11-15T17:40:30,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/8a9e9d2b10b3449480da3f393ac19e1b, entries=150, sequenceid=334, filesize=30.5 K 2024-11-15T17:40:30,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/bd9a91e5df39428690379202d4bf88e2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/bd9a91e5df39428690379202d4bf88e2 2024-11-15T17:40:30,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/bd9a91e5df39428690379202d4bf88e2, entries=150, sequenceid=334, filesize=12.0 K 2024-11-15T17:40:30,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/d5e8e8b63c914caea6095cc1a28a54e2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d5e8e8b63c914caea6095cc1a28a54e2 2024-11-15T17:40:30,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d5e8e8b63c914caea6095cc1a28a54e2, entries=150, sequenceid=334, filesize=12.0 K 2024-11-15T17:40:30,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 103dbae58ceff9ab6b93f5c0fda56a23 in 667ms, sequenceid=334, compaction requested=true 2024-11-15T17:40:30,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:30,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:30,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:30,174 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:30,174 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:30,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:30,176 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:30,176 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:30,176 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:30,176 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:30,176 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,176 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,176 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c54703d276864aefadb06e76dfb9d601, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4cacb1c63e79477aa990ba98fe1bb00e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/bd9a91e5df39428690379202d4bf88e2] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=36.7 K 2024-11-15T17:40:30,176 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0bed777d379c43118661a5358589377e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/73788571b29a4a3d9384548b95313820, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/8a9e9d2b10b3449480da3f393ac19e1b] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=92.2 K 2024-11-15T17:40:30,176 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,177 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0bed777d379c43118661a5358589377e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/73788571b29a4a3d9384548b95313820, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/8a9e9d2b10b3449480da3f393ac19e1b] 2024-11-15T17:40:30,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:30,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:30,177 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c54703d276864aefadb06e76dfb9d601, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731692425771 2024-11-15T17:40:30,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:30,177 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bed777d379c43118661a5358589377e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731692425771 2024-11-15T17:40:30,178 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cacb1c63e79477aa990ba98fe1bb00e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731692426998 2024-11-15T17:40:30,178 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73788571b29a4a3d9384548b95313820, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731692426998 2024-11-15T17:40:30,178 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9a91e5df39428690379202d4bf88e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731692429187 2024-11-15T17:40:30,178 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a9e9d2b10b3449480da3f393ac19e1b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731692429187 2024-11-15T17:40:30,191 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:30,193 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:30,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:40:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:30,201 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:30,202 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/336e64fa9f5e4b6a927a1c77e5e02f82 is 50, key is test_row_0/B:col10/1731692429505/Put/seqid=0 2024-11-15T17:40:30,211 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115bfecdad1533a4732a57a5c442ff9d64a_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:30,213 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115bfecdad1533a4732a57a5c442ff9d64a_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:30,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bfecdad1533a4732a57a5c442ff9d64a_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692490220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692490223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692490227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692490229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692490234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T17:40:30,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:30,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742033_1209 (size=13085) 2024-11-15T17:40:30,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150c583b408d2f4b7594e0d396bca43b9f_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692430191/Put/seqid=0 2024-11-15T17:40:30,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742034_1210 (size=4469) 2024-11-15T17:40:30,296 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#170 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:30,297 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/eebc997196fd42efb2909d2327b2155c is 175, key is test_row_0/A:col10/1731692429505/Put/seqid=0 2024-11-15T17:40:30,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742036_1212 (size=32039) 2024-11-15T17:40:30,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742035_1211 (size=12454) 2024-11-15T17:40:30,335 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:30,342 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150c583b408d2f4b7594e0d396bca43b9f_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150c583b408d2f4b7594e0d396bca43b9f_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:30,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/7a7c7c351f194466831499bed5fc09af, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:30,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/7a7c7c351f194466831499bed5fc09af is 175, key is test_row_0/A:col10/1731692430191/Put/seqid=0 2024-11-15T17:40:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692490345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692490345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692490345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692490345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692490355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742037_1213 (size=31255) 2024-11-15T17:40:30,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/7a7c7c351f194466831499bed5fc09af 2024-11-15T17:40:30,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/3ea00b82fa9a40e7817528e686d94447 is 50, key is test_row_0/B:col10/1731692430191/Put/seqid=0 2024-11-15T17:40:30,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742038_1214 (size=12301) 2024-11-15T17:40:30,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/3ea00b82fa9a40e7817528e686d94447 2024-11-15T17:40:30,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T17:40:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T17:40:30,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/734bb323b95646a69faaabd09dac68c5 is 50, key is test_row_0/C:col10/1731692430191/Put/seqid=0 2024-11-15T17:40:30,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742039_1215 (size=12301) 2024-11-15T17:40:30,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692490556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692490556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692490557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T17:40:30,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692490557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692490567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,679 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/336e64fa9f5e4b6a927a1c77e5e02f82 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/336e64fa9f5e4b6a927a1c77e5e02f82 2024-11-15T17:40:30,686 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into 336e64fa9f5e4b6a927a1c77e5e02f82(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:30,686 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:30,686 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=13, startTime=1731692430174; duration=0sec 2024-11-15T17:40:30,686 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:30,686 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:30,686 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:30,687 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:30,688 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/C is initiating minor compaction (all files) 2024-11-15T17:40:30,688 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/C in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,688 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/38bc57d763ce422d98f14de6fe84037f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/53b180082bd04082850279e19148cb43, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d5e8e8b63c914caea6095cc1a28a54e2] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=36.7 K 2024-11-15T17:40:30,688 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 38bc57d763ce422d98f14de6fe84037f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731692425771 2024-11-15T17:40:30,689 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b180082bd04082850279e19148cb43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731692426998 2024-11-15T17:40:30,689 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d5e8e8b63c914caea6095cc1a28a54e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731692429187 2024-11-15T17:40:30,701 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#C#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:30,701 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/76c417da80bd482d9b8346e7c539786c is 50, key is test_row_0/C:col10/1731692429505/Put/seqid=0 2024-11-15T17:40:30,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742040_1216 (size=13051) 2024-11-15T17:40:30,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T17:40:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:30,731 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/eebc997196fd42efb2909d2327b2155c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/eebc997196fd42efb2909d2327b2155c 2024-11-15T17:40:30,739 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into eebc997196fd42efb2909d2327b2155c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:30,739 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:30,739 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=13, startTime=1731692430174; duration=0sec 2024-11-15T17:40:30,739 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:30,739 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:30,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/734bb323b95646a69faaabd09dac68c5 2024-11-15T17:40:30,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/7a7c7c351f194466831499bed5fc09af as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/7a7c7c351f194466831499bed5fc09af 2024-11-15T17:40:30,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/7a7c7c351f194466831499bed5fc09af, entries=150, sequenceid=355, filesize=30.5 K 2024-11-15T17:40:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/3ea00b82fa9a40e7817528e686d94447 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/3ea00b82fa9a40e7817528e686d94447 2024-11-15T17:40:30,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/3ea00b82fa9a40e7817528e686d94447, entries=150, sequenceid=355, filesize=12.0 K 2024-11-15T17:40:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/734bb323b95646a69faaabd09dac68c5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/734bb323b95646a69faaabd09dac68c5 2024-11-15T17:40:30,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/734bb323b95646a69faaabd09dac68c5, entries=150, sequenceid=355, filesize=12.0 K 2024-11-15T17:40:30,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 103dbae58ceff9ab6b93f5c0fda56a23 in 667ms, sequenceid=355, compaction requested=false 2024-11-15T17:40:30,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:30,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T17:40:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:30,873 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T17:40:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:30,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115523b4ad668cf4606ac54330d8afeb539_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692430227/Put/seqid=0 2024-11-15T17:40:30,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742041_1217 (size=12454) 2024-11-15T17:40:30,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692490900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692490900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692490901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692490902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692490903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T17:40:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692491006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692491015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692491015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692491015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692491015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,120 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/76c417da80bd482d9b8346e7c539786c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/76c417da80bd482d9b8346e7c539786c 2024-11-15T17:40:31,125 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/C of 103dbae58ceff9ab6b93f5c0fda56a23 into 76c417da80bd482d9b8346e7c539786c(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:31,125 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:31,125 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/C, priority=13, startTime=1731692430177; duration=0sec 2024-11-15T17:40:31,125 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:31,125 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:31,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692491216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692491225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692491225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692491225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692491225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:31,303 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115523b4ad668cf4606ac54330d8afeb539_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115523b4ad668cf4606ac54330d8afeb539_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/0d3044552d824b83afe14aec9a0f8a52, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/0d3044552d824b83afe14aec9a0f8a52 is 175, key is test_row_0/A:col10/1731692430227/Put/seqid=0 2024-11-15T17:40:31,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742042_1218 (size=31255) 2024-11-15T17:40:31,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692491526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692491545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692491545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692491545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692491545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:31,719 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/0d3044552d824b83afe14aec9a0f8a52 2024-11-15T17:40:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/b103e6067b7740b7bb08cf934b13876f is 50, key is test_row_0/B:col10/1731692430227/Put/seqid=0 2024-11-15T17:40:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742043_1219 (size=12301) 2024-11-15T17:40:31,755 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/b103e6067b7740b7bb08cf934b13876f 2024-11-15T17:40:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/7010d05c7c6949a09f1a631bb84541cf is 50, key is test_row_0/C:col10/1731692430227/Put/seqid=0 2024-11-15T17:40:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742044_1220 (size=12301) 2024-11-15T17:40:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T17:40:32,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692492035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692492056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692492056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692492056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692492057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:32,189 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/7010d05c7c6949a09f1a631bb84541cf 2024-11-15T17:40:32,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/0d3044552d824b83afe14aec9a0f8a52 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0d3044552d824b83afe14aec9a0f8a52 2024-11-15T17:40:32,199 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0d3044552d824b83afe14aec9a0f8a52, entries=150, sequenceid=372, filesize=30.5 K 2024-11-15T17:40:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/b103e6067b7740b7bb08cf934b13876f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b103e6067b7740b7bb08cf934b13876f 2024-11-15T17:40:32,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b103e6067b7740b7bb08cf934b13876f, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T17:40:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/7010d05c7c6949a09f1a631bb84541cf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/7010d05c7c6949a09f1a631bb84541cf 2024-11-15T17:40:32,217 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/7010d05c7c6949a09f1a631bb84541cf, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T17:40:32,218 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1345ms, sequenceid=372, compaction requested=true 2024-11-15T17:40:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:32,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:32,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T17:40:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T17:40:32,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T17:40:32,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4290 sec 2024-11-15T17:40:32,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4370 sec 2024-11-15T17:40:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:33,048 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:33,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T17:40:33,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c990eee45c4f426092ae477cacb6a11a_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692433047/Put/seqid=0 2024-11-15T17:40:33,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742045_1221 (size=14994) 2024-11-15T17:40:33,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692493083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692493084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692493085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692493086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692493087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692493195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692493195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692493195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692493195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692493196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692493406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692493406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692493406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692493406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692493406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:33,488 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c990eee45c4f426092ae477cacb6a11a_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c990eee45c4f426092ae477cacb6a11a_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:33,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ce2307c083504975ad1ac0b09233a519, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:33,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ce2307c083504975ad1ac0b09233a519 is 175, key is test_row_0/A:col10/1731692433047/Put/seqid=0 2024-11-15T17:40:33,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742046_1222 (size=39949) 2024-11-15T17:40:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692493715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692493715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692493715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692493716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692493716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:33,897 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ce2307c083504975ad1ac0b09233a519 2024-11-15T17:40:33,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/0dcc73bd466e43e38111aa4fdce25085 is 50, key is test_row_0/B:col10/1731692433047/Put/seqid=0 2024-11-15T17:40:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T17:40:33,967 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742047_1223 (size=12301) 2024-11-15T17:40:33,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:33,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/0dcc73bd466e43e38111aa4fdce25085 2024-11-15T17:40:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-15T17:40:33,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T17:40:33,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:33,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:33,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/6401c0793681498fbbbeac089abcf604 is 50, key is test_row_0/C:col10/1731692433047/Put/seqid=0 2024-11-15T17:40:34,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742048_1224 (size=12301) 2024-11-15T17:40:34,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/6401c0793681498fbbbeac089abcf604 2024-11-15T17:40:34,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ce2307c083504975ad1ac0b09233a519 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ce2307c083504975ad1ac0b09233a519 2024-11-15T17:40:34,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ce2307c083504975ad1ac0b09233a519, entries=200, sequenceid=395, filesize=39.0 K 2024-11-15T17:40:34,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/0dcc73bd466e43e38111aa4fdce25085 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/0dcc73bd466e43e38111aa4fdce25085 2024-11-15T17:40:34,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/0dcc73bd466e43e38111aa4fdce25085, entries=150, sequenceid=395, filesize=12.0 K 2024-11-15T17:40:34,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/6401c0793681498fbbbeac089abcf604 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/6401c0793681498fbbbeac089abcf604 2024-11-15T17:40:34,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/6401c0793681498fbbbeac089abcf604, entries=150, sequenceid=395, filesize=12.0 K 2024-11-15T17:40:34,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1004ms, sequenceid=395, compaction requested=true 2024-11-15T17:40:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:34,052 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:34,052 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 103dbae58ceff9ab6b93f5c0fda56a23:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:34,055 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:34,055 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/A is initiating minor compaction (all files) 2024-11-15T17:40:34,055 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/A in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:34,055 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:34,055 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/B is initiating minor compaction (all files) 2024-11-15T17:40:34,055 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/eebc997196fd42efb2909d2327b2155c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/7a7c7c351f194466831499bed5fc09af, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0d3044552d824b83afe14aec9a0f8a52, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ce2307c083504975ad1ac0b09233a519] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=131.3 K 2024-11-15T17:40:34,055 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/B in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:34,055 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:34,055 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/336e64fa9f5e4b6a927a1c77e5e02f82, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/3ea00b82fa9a40e7817528e686d94447, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b103e6067b7740b7bb08cf934b13876f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/0dcc73bd466e43e38111aa4fdce25085] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=48.8 K 2024-11-15T17:40:34,055 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/eebc997196fd42efb2909d2327b2155c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/7a7c7c351f194466831499bed5fc09af, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0d3044552d824b83afe14aec9a0f8a52, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ce2307c083504975ad1ac0b09233a519] 2024-11-15T17:40:34,056 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 336e64fa9f5e4b6a927a1c77e5e02f82, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731692429187 2024-11-15T17:40:34,056 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting eebc997196fd42efb2909d2327b2155c, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731692429187 2024-11-15T17:40:34,056 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ea00b82fa9a40e7817528e686d94447, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731692429547 2024-11-15T17:40:34,056 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7c7c351f194466831499bed5fc09af, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731692429547 2024-11-15T17:40:34,056 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b103e6067b7740b7bb08cf934b13876f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731692430221 2024-11-15T17:40:34,056 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d3044552d824b83afe14aec9a0f8a52, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731692430221 2024-11-15T17:40:34,056 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dcc73bd466e43e38111aa4fdce25085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731692430899 2024-11-15T17:40:34,057 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2307c083504975ad1ac0b09233a519, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731692430899 2024-11-15T17:40:34,068 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#B#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:34,068 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/213372609d89489d8a74458e33d9c5fa is 50, key is test_row_0/B:col10/1731692433047/Put/seqid=0 2024-11-15T17:40:34,073 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T17:40:34,102 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111516f2dfc4ee034cb0a9c7f03281c55e2b_103dbae58ceff9ab6b93f5c0fda56a23 store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:34,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742049_1225 (size=13221) 2024-11-15T17:40:34,104 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111516f2dfc4ee034cb0a9c7f03281c55e2b_103dbae58ceff9ab6b93f5c0fda56a23, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:34,105 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111516f2dfc4ee034cb0a9c7f03281c55e2b_103dbae58ceff9ab6b93f5c0fda56a23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:34,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742050_1226 (size=4469) 2024-11-15T17:40:34,125 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/213372609d89489d8a74458e33d9c5fa as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/213372609d89489d8a74458e33d9c5fa 2024-11-15T17:40:34,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T17:40:34,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:34,130 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T17:40:34,134 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/B of 103dbae58ceff9ab6b93f5c0fda56a23 into 213372609d89489d8a74458e33d9c5fa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:34,134 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:34,134 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/B, priority=12, startTime=1731692434052; duration=0sec 2024-11-15T17:40:34,134 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:34,134 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:B 2024-11-15T17:40:34,134 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:34,136 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:34,136 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 103dbae58ceff9ab6b93f5c0fda56a23/C is initiating minor compaction (all files) 2024-11-15T17:40:34,136 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 103dbae58ceff9ab6b93f5c0fda56a23/C in TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:34,136 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/76c417da80bd482d9b8346e7c539786c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/734bb323b95646a69faaabd09dac68c5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/7010d05c7c6949a09f1a631bb84541cf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/6401c0793681498fbbbeac089abcf604] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp, totalSize=48.8 K 2024-11-15T17:40:34,136 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c417da80bd482d9b8346e7c539786c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731692429187 2024-11-15T17:40:34,137 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 734bb323b95646a69faaabd09dac68c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731692429547 2024-11-15T17:40:34,137 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7010d05c7c6949a09f1a631bb84541cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731692430221 2024-11-15T17:40:34,137 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6401c0793681498fbbbeac089abcf604, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731692430899 2024-11-15T17:40:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115811304d32f2c43b9b793caa7b9149e9d_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692433084/Put/seqid=0 2024-11-15T17:40:34,147 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#C#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:34,147 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/610be4d6f26f499ea2a9e1e6e9f1859e is 50, key is test_row_0/C:col10/1731692433047/Put/seqid=0 2024-11-15T17:40:34,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742051_1227 (size=12454) 2024-11-15T17:40:34,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:34,160 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115811304d32f2c43b9b793caa7b9149e9d_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115811304d32f2c43b9b793caa7b9149e9d_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:34,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d2c90382fd7e4549afde67e7739eabdf, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:34,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d2c90382fd7e4549afde67e7739eabdf is 175, key is test_row_0/A:col10/1731692433084/Put/seqid=0 2024-11-15T17:40:34,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742052_1228 (size=13187) 2024-11-15T17:40:34,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742053_1229 (size=31255) 2024-11-15T17:40:34,175 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/610be4d6f26f499ea2a9e1e6e9f1859e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/610be4d6f26f499ea2a9e1e6e9f1859e 2024-11-15T17:40:34,181 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/C of 103dbae58ceff9ab6b93f5c0fda56a23 into 610be4d6f26f499ea2a9e1e6e9f1859e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:34,181 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:34,181 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/C, priority=12, startTime=1731692434052; duration=0sec 2024-11-15T17:40:34,181 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:34,181 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:C 2024-11-15T17:40:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692494248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692494251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692494252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692494253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692494254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T17:40:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692494355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692494355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692494366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692494366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692494371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,525 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 103dbae58ceff9ab6b93f5c0fda56a23#A#compaction#183 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:34,526 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ec486dbdf39541d9b99486ab746dab12 is 175, key is test_row_0/A:col10/1731692433047/Put/seqid=0 2024-11-15T17:40:34,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742054_1230 (size=32175) 2024-11-15T17:40:34,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692494566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692494566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,569 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d2c90382fd7e4549afde67e7739eabdf 2024-11-15T17:40:34,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692494576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692494575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692494576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/ec195e37de90420c8d1493dc6aa4dd56 is 50, key is test_row_0/B:col10/1731692433084/Put/seqid=0 2024-11-15T17:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T17:40:34,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742055_1231 (size=12301) 2024-11-15T17:40:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692494875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692494875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692494885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692494885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692494896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:34,956 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/ec486dbdf39541d9b99486ab746dab12 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ec486dbdf39541d9b99486ab746dab12 2024-11-15T17:40:34,963 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 103dbae58ceff9ab6b93f5c0fda56a23/A of 103dbae58ceff9ab6b93f5c0fda56a23 into ec486dbdf39541d9b99486ab746dab12(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:34,964 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:34,964 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., storeName=103dbae58ceff9ab6b93f5c0fda56a23/A, priority=12, startTime=1731692434052; duration=0sec 2024-11-15T17:40:34,964 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:34,964 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 103dbae58ceff9ab6b93f5c0fda56a23:A 2024-11-15T17:40:35,037 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/ec195e37de90420c8d1493dc6aa4dd56 2024-11-15T17:40:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/db5c019143444d009f6b274774a3245a is 50, key is test_row_0/C:col10/1731692433084/Put/seqid=0 2024-11-15T17:40:35,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742056_1232 (size=12301) 2024-11-15T17:40:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T17:40:35,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:35,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51854 deadline: 1731692495385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51910 deadline: 1731692495385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:35,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731692495395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1731692495395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51824 deadline: 1731692495406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:40:35,459 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/db5c019143444d009f6b274774a3245a 2024-11-15T17:40:35,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/d2c90382fd7e4549afde67e7739eabdf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d2c90382fd7e4549afde67e7739eabdf 2024-11-15T17:40:35,469 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d2c90382fd7e4549afde67e7739eabdf, entries=150, sequenceid=410, filesize=30.5 K 2024-11-15T17:40:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/ec195e37de90420c8d1493dc6aa4dd56 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/ec195e37de90420c8d1493dc6aa4dd56 2024-11-15T17:40:35,475 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/ec195e37de90420c8d1493dc6aa4dd56, entries=150, sequenceid=410, filesize=12.0 K 2024-11-15T17:40:35,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/db5c019143444d009f6b274774a3245a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/db5c019143444d009f6b274774a3245a 2024-11-15T17:40:35,484 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/db5c019143444d009f6b274774a3245a, entries=150, sequenceid=410, filesize=12.0 K 2024-11-15T17:40:35,485 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 103dbae58ceff9ab6b93f5c0fda56a23 in 1355ms, sequenceid=410, compaction requested=false 2024-11-15T17:40:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-15T17:40:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-15T17:40:35,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T17:40:35,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-11-15T17:40:35,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5220 sec 2024-11-15T17:40:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T17:40:36,136 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:36,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-15T17:40:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T17:40:36,138 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:36,139 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:36,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T17:40:36,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T17:40:36,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:36,292 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T17:40:36,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e387fd41f004a709871f9f537307837_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692434251/Put/seqid=0 2024-11-15T17:40:36,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742057_1233 (size=12454) 2024-11-15T17:40:36,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:36,315 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e387fd41f004a709871f9f537307837_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e387fd41f004a709871f9f537307837_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/dd22c5923b124a2cbfb5ccaad36ffc75, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/dd22c5923b124a2cbfb5ccaad36ffc75 is 175, key is test_row_0/A:col10/1731692434251/Put/seqid=0 2024-11-15T17:40:36,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742058_1234 (size=31255) 2024-11-15T17:40:36,333 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/dd22c5923b124a2cbfb5ccaad36ffc75 2024-11-15T17:40:36,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/1fb9ad2eb513469283f11a849a7e7778 is 50, key is test_row_0/B:col10/1731692434251/Put/seqid=0 2024-11-15T17:40:36,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742059_1235 (size=12301) 2024-11-15T17:40:36,391 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-11-15T17:40:36,391 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,391 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,392 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,393 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-11-15T17:40:36,393 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,393 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,393 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,393 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-11-15T17:40:36,393 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,393 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,393 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,393 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-15T17:40:36,394 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,394 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,394 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:36,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. as already flushing 2024-11-15T17:40:36,396 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-15T17:40:36,396 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,396 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,396 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,407 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-15T17:40:36,407 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-15T17:40:36,407 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,407 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,407 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,407 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,407 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,407 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,407 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-15T17:40:36,407 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,407 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,407 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,417 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-15T17:40:36,417 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:40:36,418 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,418 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:36,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T17:40:36,749 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/1fb9ad2eb513469283f11a849a7e7778 2024-11-15T17:40:36,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/19b13ed78ee14b838d518e9e84bbf9aa is 50, key is test_row_0/C:col10/1731692434251/Put/seqid=0 2024-11-15T17:40:36,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742060_1236 (size=12301) 2024-11-15T17:40:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T17:40:37,162 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/19b13ed78ee14b838d518e9e84bbf9aa 2024-11-15T17:40:37,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/dd22c5923b124a2cbfb5ccaad36ffc75 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/dd22c5923b124a2cbfb5ccaad36ffc75 2024-11-15T17:40:37,172 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/dd22c5923b124a2cbfb5ccaad36ffc75, entries=150, sequenceid=434, filesize=30.5 K 2024-11-15T17:40:37,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/1fb9ad2eb513469283f11a849a7e7778 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/1fb9ad2eb513469283f11a849a7e7778 2024-11-15T17:40:37,177 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/1fb9ad2eb513469283f11a849a7e7778, entries=150, sequenceid=434, filesize=12.0 K 2024-11-15T17:40:37,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/19b13ed78ee14b838d518e9e84bbf9aa as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/19b13ed78ee14b838d518e9e84bbf9aa 2024-11-15T17:40:37,183 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/19b13ed78ee14b838d518e9e84bbf9aa, entries=150, sequenceid=434, filesize=12.0 K 2024-11-15T17:40:37,183 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=33.54 KB/34350 for 103dbae58ceff9ab6b93f5c0fda56a23 in 891ms, sequenceid=434, compaction requested=true 2024-11-15T17:40:37,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 103dbae58ceff9ab6b93f5c0fda56a23: 2024-11-15T17:40:37,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:37,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-15T17:40:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-15T17:40:37,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T17:40:37,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0460 sec 2024-11-15T17:40:37,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.0510 sec 2024-11-15T17:40:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T17:40:37,276 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:37,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2875 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2832 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1302 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3905 rows 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1298 2024-11-15T17:40:37,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3893 rows 2024-11-15T17:40:37,277 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:40:37,277 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:40:37,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:37,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:37,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:37,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:40:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T17:40:37,281 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692437281"}]},"ts":"1731692437281"} 2024-11-15T17:40:37,282 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:40:37,282 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:40:37,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:40:37,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, UNASSIGN}] 2024-11-15T17:40:37,285 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, UNASSIGN 2024-11-15T17:40:37,285 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:37,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, UNASSIGN because future has completed 2024-11-15T17:40:37,287 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:40:37,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:40:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T17:40:37,440 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:37,440 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:40:37,440 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 103dbae58ceff9ab6b93f5c0fda56a23, disabling compactions & flushes 2024-11-15T17:40:37,440 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:37,440 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:37,440 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. after waiting 0 ms 2024-11-15T17:40:37,440 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:37,440 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 103dbae58ceff9ab6b93f5c0fda56a23 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T17:40:37,445 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159edb33f479e74edf873871a8d102ad68_103dbae58ceff9ab6b93f5c0fda56a23 is 50, key is test_row_0/A:col10/1731692436416/Put/seqid=0 2024-11-15T17:40:37,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742061_1237 (size=9914) 2024-11-15T17:40:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T17:40:37,850 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:37,853 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159edb33f479e74edf873871a8d102ad68_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159edb33f479e74edf873871a8d102ad68_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:37,854 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bef1136972714eac852aaf4cdf045b16, store: [table=TestAcidGuarantees family=A region=103dbae58ceff9ab6b93f5c0fda56a23] 2024-11-15T17:40:37,855 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bef1136972714eac852aaf4cdf045b16 is 175, key is test_row_0/A:col10/1731692436416/Put/seqid=0 2024-11-15T17:40:37,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742062_1238 (size=22561) 2024-11-15T17:40:37,864 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bef1136972714eac852aaf4cdf045b16 2024-11-15T17:40:37,878 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/992435ef56f44231828e2ab15b81a2f6 is 50, key is test_row_0/B:col10/1731692436416/Put/seqid=0 2024-11-15T17:40:37,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742063_1239 (size=9857) 2024-11-15T17:40:37,891 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/992435ef56f44231828e2ab15b81a2f6 2024-11-15T17:40:37,899 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/cf4b2862393541e48d8fed41de9e9f47 is 50, key is test_row_0/C:col10/1731692436416/Put/seqid=0 2024-11-15T17:40:37,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742064_1240 (size=9857) 2024-11-15T17:40:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T17:40:38,305 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/cf4b2862393541e48d8fed41de9e9f47 2024-11-15T17:40:38,312 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/A/bef1136972714eac852aaf4cdf045b16 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bef1136972714eac852aaf4cdf045b16 2024-11-15T17:40:38,317 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bef1136972714eac852aaf4cdf045b16, entries=100, sequenceid=442, filesize=22.0 K 2024-11-15T17:40:38,318 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/B/992435ef56f44231828e2ab15b81a2f6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/992435ef56f44231828e2ab15b81a2f6 2024-11-15T17:40:38,323 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/992435ef56f44231828e2ab15b81a2f6, entries=100, sequenceid=442, filesize=9.6 K 2024-11-15T17:40:38,324 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/.tmp/C/cf4b2862393541e48d8fed41de9e9f47 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/cf4b2862393541e48d8fed41de9e9f47 2024-11-15T17:40:38,328 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/cf4b2862393541e48d8fed41de9e9f47, entries=100, sequenceid=442, filesize=9.6 K 2024-11-15T17:40:38,328 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 103dbae58ceff9ab6b93f5c0fda56a23 in 888ms, sequenceid=442, compaction requested=true 2024-11-15T17:40:38,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/fa7a437076504d6a9e2f017e636940db, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d540f18193dc4ee68088f17663593567, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc1b4302a1b3430693cf5461cad53343, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f5d2fcdde5a44164add5b15f0932abe5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc62cb491a38441a915b9acd5e7f7511, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/c101e7370eec4e4980131f65e66dfeba, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3db5f30fddef4e6ba2dab30f6dc1e42b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f4090f19e7f341c398430a787d6ce995, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ecc0878b7b0a4cddbd7b54c1a50ebed4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bf52cf93fcd844ee9f9f15b09c4a2365, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3e3211c9749f4068b73eb80b26de94b3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/4dce059031e54d8e811f2fcc9bcc6fed, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/03c36bc04c4446aabe0e731a25542017, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/cd8b7305e6fe41eeb7c6ac40c284879b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/27c7b7899c964a66a4a9c55fd7191d37, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ae7c5f7e115a4dddafb56593369fdcd2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/46fb930fb6fd4bd0aa650a37326b00ab, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/43931d4fac4249efb4505c01ed100542, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bd1a26e3dd334def90281a832a1a4891, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0bed777d379c43118661a5358589377e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f97ea72ee7d54ccdafc8bcbfbf20deb2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/73788571b29a4a3d9384548b95313820, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/eebc997196fd42efb2909d2327b2155c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/8a9e9d2b10b3449480da3f393ac19e1b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/7a7c7c351f194466831499bed5fc09af, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0d3044552d824b83afe14aec9a0f8a52, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ce2307c083504975ad1ac0b09233a519] to archive 2024-11-15T17:40:38,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:40:38,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/fa7a437076504d6a9e2f017e636940db to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/fa7a437076504d6a9e2f017e636940db 2024-11-15T17:40:38,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d540f18193dc4ee68088f17663593567 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d540f18193dc4ee68088f17663593567 2024-11-15T17:40:38,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc1b4302a1b3430693cf5461cad53343 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc1b4302a1b3430693cf5461cad53343 2024-11-15T17:40:38,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f5d2fcdde5a44164add5b15f0932abe5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f5d2fcdde5a44164add5b15f0932abe5 2024-11-15T17:40:38,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc62cb491a38441a915b9acd5e7f7511 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bc62cb491a38441a915b9acd5e7f7511 2024-11-15T17:40:38,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/c101e7370eec4e4980131f65e66dfeba to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/c101e7370eec4e4980131f65e66dfeba 2024-11-15T17:40:38,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3db5f30fddef4e6ba2dab30f6dc1e42b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3db5f30fddef4e6ba2dab30f6dc1e42b 2024-11-15T17:40:38,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f4090f19e7f341c398430a787d6ce995 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f4090f19e7f341c398430a787d6ce995 2024-11-15T17:40:38,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ecc0878b7b0a4cddbd7b54c1a50ebed4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ecc0878b7b0a4cddbd7b54c1a50ebed4 2024-11-15T17:40:38,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bf52cf93fcd844ee9f9f15b09c4a2365 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bf52cf93fcd844ee9f9f15b09c4a2365 2024-11-15T17:40:38,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3e3211c9749f4068b73eb80b26de94b3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/3e3211c9749f4068b73eb80b26de94b3 2024-11-15T17:40:38,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/4dce059031e54d8e811f2fcc9bcc6fed to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/4dce059031e54d8e811f2fcc9bcc6fed 2024-11-15T17:40:38,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/03c36bc04c4446aabe0e731a25542017 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/03c36bc04c4446aabe0e731a25542017 2024-11-15T17:40:38,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/cd8b7305e6fe41eeb7c6ac40c284879b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/cd8b7305e6fe41eeb7c6ac40c284879b 2024-11-15T17:40:38,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/27c7b7899c964a66a4a9c55fd7191d37 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/27c7b7899c964a66a4a9c55fd7191d37 2024-11-15T17:40:38,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ae7c5f7e115a4dddafb56593369fdcd2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ae7c5f7e115a4dddafb56593369fdcd2 2024-11-15T17:40:38,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/46fb930fb6fd4bd0aa650a37326b00ab to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/46fb930fb6fd4bd0aa650a37326b00ab 2024-11-15T17:40:38,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/43931d4fac4249efb4505c01ed100542 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/43931d4fac4249efb4505c01ed100542 2024-11-15T17:40:38,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bd1a26e3dd334def90281a832a1a4891 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bd1a26e3dd334def90281a832a1a4891 2024-11-15T17:40:38,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0bed777d379c43118661a5358589377e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0bed777d379c43118661a5358589377e 2024-11-15T17:40:38,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f97ea72ee7d54ccdafc8bcbfbf20deb2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/f97ea72ee7d54ccdafc8bcbfbf20deb2 2024-11-15T17:40:38,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/73788571b29a4a3d9384548b95313820 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/73788571b29a4a3d9384548b95313820 2024-11-15T17:40:38,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/eebc997196fd42efb2909d2327b2155c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/eebc997196fd42efb2909d2327b2155c 2024-11-15T17:40:38,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/8a9e9d2b10b3449480da3f393ac19e1b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/8a9e9d2b10b3449480da3f393ac19e1b 2024-11-15T17:40:38,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/7a7c7c351f194466831499bed5fc09af to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/7a7c7c351f194466831499bed5fc09af 2024-11-15T17:40:38,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0d3044552d824b83afe14aec9a0f8a52 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/0d3044552d824b83afe14aec9a0f8a52 2024-11-15T17:40:38,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ce2307c083504975ad1ac0b09233a519 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ce2307c083504975ad1ac0b09233a519 2024-11-15T17:40:38,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fc71ce1aded74cf286a2295cb991d2ea, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/58275dd8034546dab05b3b53ffd66c44, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/64b059bd060a47989d2983327d133a54, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/716ebe1fbd7f40b6aaf1dc81520d40c5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/da8024f7ae1047bb9b5f66a0f2205988, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c87431cfe9de400c9225555f108eaec3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f2c739ea0a25484a8cecf19655f431ac, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/31e31320203e4d688497f359971fc50d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/dd299ab3c4414869860fc768784acddf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b473a56c86a54c02a986d8e55c43f2d6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/790e250f9ba44ee29b7037f9c5164f07, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/5225e171f3794a848b1d6a24aab1f087, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c12543d9ff1b4092ab37f2fcc191f489, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fe043b5afc314a77bc4d41831dbf2123, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d07c7d260d7941fbb31b3764d432de5e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d1ee2c791b2845c0a96e566e4c29c982, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/6ddfeafd623f4786aa45b4d79d4a4f6e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f31a85b7bbd3486e82c531ffc6b28265, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/72f7c7f8dffe423c908cb1072ce5622e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c54703d276864aefadb06e76dfb9d601, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4d43930a894342ddb73c5023ddb52be5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4cacb1c63e79477aa990ba98fe1bb00e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/336e64fa9f5e4b6a927a1c77e5e02f82, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/bd9a91e5df39428690379202d4bf88e2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/3ea00b82fa9a40e7817528e686d94447, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b103e6067b7740b7bb08cf934b13876f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/0dcc73bd466e43e38111aa4fdce25085] to archive 2024-11-15T17:40:38,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:40:38,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fc71ce1aded74cf286a2295cb991d2ea to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fc71ce1aded74cf286a2295cb991d2ea 2024-11-15T17:40:38,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/58275dd8034546dab05b3b53ffd66c44 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/58275dd8034546dab05b3b53ffd66c44 2024-11-15T17:40:38,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/64b059bd060a47989d2983327d133a54 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/64b059bd060a47989d2983327d133a54 2024-11-15T17:40:38,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/716ebe1fbd7f40b6aaf1dc81520d40c5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/716ebe1fbd7f40b6aaf1dc81520d40c5 2024-11-15T17:40:38,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/da8024f7ae1047bb9b5f66a0f2205988 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/da8024f7ae1047bb9b5f66a0f2205988 2024-11-15T17:40:38,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c87431cfe9de400c9225555f108eaec3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c87431cfe9de400c9225555f108eaec3 2024-11-15T17:40:38,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f2c739ea0a25484a8cecf19655f431ac to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f2c739ea0a25484a8cecf19655f431ac 2024-11-15T17:40:38,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/31e31320203e4d688497f359971fc50d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/31e31320203e4d688497f359971fc50d 2024-11-15T17:40:38,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/dd299ab3c4414869860fc768784acddf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/dd299ab3c4414869860fc768784acddf 2024-11-15T17:40:38,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b473a56c86a54c02a986d8e55c43f2d6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b473a56c86a54c02a986d8e55c43f2d6 2024-11-15T17:40:38,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/790e250f9ba44ee29b7037f9c5164f07 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/790e250f9ba44ee29b7037f9c5164f07 2024-11-15T17:40:38,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/5225e171f3794a848b1d6a24aab1f087 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/5225e171f3794a848b1d6a24aab1f087 2024-11-15T17:40:38,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c12543d9ff1b4092ab37f2fcc191f489 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c12543d9ff1b4092ab37f2fcc191f489 2024-11-15T17:40:38,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fe043b5afc314a77bc4d41831dbf2123 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/fe043b5afc314a77bc4d41831dbf2123 2024-11-15T17:40:38,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d07c7d260d7941fbb31b3764d432de5e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d07c7d260d7941fbb31b3764d432de5e 2024-11-15T17:40:38,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d1ee2c791b2845c0a96e566e4c29c982 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/d1ee2c791b2845c0a96e566e4c29c982 2024-11-15T17:40:38,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/6ddfeafd623f4786aa45b4d79d4a4f6e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/6ddfeafd623f4786aa45b4d79d4a4f6e 2024-11-15T17:40:38,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f31a85b7bbd3486e82c531ffc6b28265 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/f31a85b7bbd3486e82c531ffc6b28265 2024-11-15T17:40:38,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/72f7c7f8dffe423c908cb1072ce5622e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/72f7c7f8dffe423c908cb1072ce5622e 2024-11-15T17:40:38,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c54703d276864aefadb06e76dfb9d601 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/c54703d276864aefadb06e76dfb9d601 2024-11-15T17:40:38,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4d43930a894342ddb73c5023ddb52be5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4d43930a894342ddb73c5023ddb52be5 2024-11-15T17:40:38,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4cacb1c63e79477aa990ba98fe1bb00e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/4cacb1c63e79477aa990ba98fe1bb00e 2024-11-15T17:40:38,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/336e64fa9f5e4b6a927a1c77e5e02f82 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/336e64fa9f5e4b6a927a1c77e5e02f82 2024-11-15T17:40:38,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/bd9a91e5df39428690379202d4bf88e2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/bd9a91e5df39428690379202d4bf88e2 2024-11-15T17:40:38,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/3ea00b82fa9a40e7817528e686d94447 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/3ea00b82fa9a40e7817528e686d94447 2024-11-15T17:40:38,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b103e6067b7740b7bb08cf934b13876f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/b103e6067b7740b7bb08cf934b13876f 2024-11-15T17:40:38,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/0dcc73bd466e43e38111aa4fdce25085 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/0dcc73bd466e43e38111aa4fdce25085 2024-11-15T17:40:38,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/68fe7fa3a3044a8888918c6c723b4801, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/86232356a4fe4dc08936740e0c951bd8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d8e02ccd226548d38abd6f408ffc67f9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/709be03a64aa45f3b77aa356e261ed30, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/ad654418c44742caa536c8e3bc1f16ed, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d767cfa4115b4316ab05ba97c8854028, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4c905e9263584eac9aef62be5a5b3f89, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/05a3329c2cef4e0baab1232750f46eb7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/49be2912ebb142a78a34f0fc83b8704e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/1d87d78f2b3e4ae0b3bca8b067a16507, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/362b042b0c894aaa8668ec7cbaf00904, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4828b869d63147bfb184d175606c01da, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/203d3fd420cc423f9dafce95b9d1ad00, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d4a0dfbd85794fd597f9886065896ffd, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/8b4479ca54ea4103ba738f58049ceb74, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/be7af70a70f84f09a3fc97bd14749a2f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/a34b3eea54bb4fa2ae47918b9044c364, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4276c0d2a8544318807127e1a4c2b312, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/38bc57d763ce422d98f14de6fe84037f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/c2378223f7f34df3ab49b6e1224838c2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/53b180082bd04082850279e19148cb43, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/76c417da80bd482d9b8346e7c539786c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d5e8e8b63c914caea6095cc1a28a54e2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/734bb323b95646a69faaabd09dac68c5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/7010d05c7c6949a09f1a631bb84541cf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/6401c0793681498fbbbeac089abcf604] to archive 2024-11-15T17:40:38,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:40:38,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/68fe7fa3a3044a8888918c6c723b4801 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/68fe7fa3a3044a8888918c6c723b4801 2024-11-15T17:40:38,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/86232356a4fe4dc08936740e0c951bd8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/86232356a4fe4dc08936740e0c951bd8 2024-11-15T17:40:38,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d8e02ccd226548d38abd6f408ffc67f9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d8e02ccd226548d38abd6f408ffc67f9 2024-11-15T17:40:38,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/709be03a64aa45f3b77aa356e261ed30 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/709be03a64aa45f3b77aa356e261ed30 2024-11-15T17:40:38,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/ad654418c44742caa536c8e3bc1f16ed to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/ad654418c44742caa536c8e3bc1f16ed 2024-11-15T17:40:38,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d767cfa4115b4316ab05ba97c8854028 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d767cfa4115b4316ab05ba97c8854028 2024-11-15T17:40:38,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4c905e9263584eac9aef62be5a5b3f89 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4c905e9263584eac9aef62be5a5b3f89 2024-11-15T17:40:38,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/05a3329c2cef4e0baab1232750f46eb7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/05a3329c2cef4e0baab1232750f46eb7 2024-11-15T17:40:38,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/49be2912ebb142a78a34f0fc83b8704e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/49be2912ebb142a78a34f0fc83b8704e 2024-11-15T17:40:38,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/1d87d78f2b3e4ae0b3bca8b067a16507 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/1d87d78f2b3e4ae0b3bca8b067a16507 2024-11-15T17:40:38,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/362b042b0c894aaa8668ec7cbaf00904 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/362b042b0c894aaa8668ec7cbaf00904 2024-11-15T17:40:38,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4828b869d63147bfb184d175606c01da to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4828b869d63147bfb184d175606c01da 2024-11-15T17:40:38,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/203d3fd420cc423f9dafce95b9d1ad00 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/203d3fd420cc423f9dafce95b9d1ad00 2024-11-15T17:40:38,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d4a0dfbd85794fd597f9886065896ffd to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d4a0dfbd85794fd597f9886065896ffd 2024-11-15T17:40:38,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/8b4479ca54ea4103ba738f58049ceb74 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/8b4479ca54ea4103ba738f58049ceb74 2024-11-15T17:40:38,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/be7af70a70f84f09a3fc97bd14749a2f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/be7af70a70f84f09a3fc97bd14749a2f 2024-11-15T17:40:38,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/a34b3eea54bb4fa2ae47918b9044c364 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/a34b3eea54bb4fa2ae47918b9044c364 2024-11-15T17:40:38,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4276c0d2a8544318807127e1a4c2b312 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/4276c0d2a8544318807127e1a4c2b312 2024-11-15T17:40:38,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/38bc57d763ce422d98f14de6fe84037f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/38bc57d763ce422d98f14de6fe84037f 2024-11-15T17:40:38,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/c2378223f7f34df3ab49b6e1224838c2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/c2378223f7f34df3ab49b6e1224838c2 2024-11-15T17:40:38,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/53b180082bd04082850279e19148cb43 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/53b180082bd04082850279e19148cb43 2024-11-15T17:40:38,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/76c417da80bd482d9b8346e7c539786c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/76c417da80bd482d9b8346e7c539786c 2024-11-15T17:40:38,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d5e8e8b63c914caea6095cc1a28a54e2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/d5e8e8b63c914caea6095cc1a28a54e2 2024-11-15T17:40:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T17:40:38,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/734bb323b95646a69faaabd09dac68c5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/734bb323b95646a69faaabd09dac68c5 2024-11-15T17:40:38,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/7010d05c7c6949a09f1a631bb84541cf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/7010d05c7c6949a09f1a631bb84541cf 2024-11-15T17:40:38,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/6401c0793681498fbbbeac089abcf604 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/6401c0793681498fbbbeac089abcf604 2024-11-15T17:40:38,422 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=4 2024-11-15T17:40:38,422 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. 2024-11-15T17:40:38,422 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 103dbae58ceff9ab6b93f5c0fda56a23: Waiting for close lock at 1731692437440Running coprocessor pre-close hooks at 1731692437440Disabling compacts and flushes for region at 1731692437440Disabling writes for close at 1731692437440Obtaining lock to block concurrent updates at 1731692437440Preparing flush snapshotting stores in 103dbae58ceff9ab6b93f5c0fda56a23 at 1731692437440Finished memstore snapshotting TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731692437440Flushing stores of TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23. at 1731692437441 (+1 ms)Flushing 103dbae58ceff9ab6b93f5c0fda56a23/A: creating writer at 1731692437441Flushing mob file 103dbae58ceff9ab6b93f5c0fda56a23/A: appending metadata at 1731692437445 (+4 ms)Flushing mob file 103dbae58ceff9ab6b93f5c0fda56a23/A: closing flushed file at 1731692437445Flushing 103dbae58ceff9ab6b93f5c0fda56a23/A: appending metadata at 1731692437854 (+409 ms)Flushing 103dbae58ceff9ab6b93f5c0fda56a23/A: closing flushed file at 1731692437855 (+1 ms)Flushing 103dbae58ceff9ab6b93f5c0fda56a23/B: creating writer at 1731692437870 (+15 ms)Flushing 103dbae58ceff9ab6b93f5c0fda56a23/B: appending metadata at 1731692437877 (+7 ms)Flushing 103dbae58ceff9ab6b93f5c0fda56a23/B: closing flushed file at 1731692437877Flushing 103dbae58ceff9ab6b93f5c0fda56a23/C: creating writer at 1731692437896 (+19 ms)Flushing 103dbae58ceff9ab6b93f5c0fda56a23/C: appending metadata at 1731692437899 (+3 ms)Flushing 103dbae58ceff9ab6b93f5c0fda56a23/C: closing flushed file at 1731692437899Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c8203e8: reopening flushed file at 1731692438310 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@410872a7: reopening flushed file at 1731692438317 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@136660dd: reopening flushed file at 1731692438323 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 103dbae58ceff9ab6b93f5c0fda56a23 in 888ms, sequenceid=442, compaction requested=true at 1731692438328 (+5 ms)Writing region close event to WAL at 1731692438419 (+91 ms)Running coprocessor post-close hooks at 1731692438422 (+3 ms)Closed at 1731692438422 2024-11-15T17:40:38,424 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:38,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=103dbae58ceff9ab6b93f5c0fda56a23, regionState=CLOSED 2024-11-15T17:40:38,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:40:38,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T17:40:38,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 103dbae58ceff9ab6b93f5c0fda56a23, server=4e3b17656000,45187,1731692383005 in 1.1390 sec 2024-11-15T17:40:38,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-15T17:40:38,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=103dbae58ceff9ab6b93f5c0fda56a23, UNASSIGN in 1.1430 sec 2024-11-15T17:40:38,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T17:40:38,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-15T17:40:38,431 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692438431"}]},"ts":"1731692438431"} 2024-11-15T17:40:38,432 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:40:38,432 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:40:38,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-15T17:40:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T17:40:39,426 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:39,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:40:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:39,428 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T17:40:39,432 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:39,435 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,438 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/recovered.edits] 2024-11-15T17:40:39,441 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bef1136972714eac852aaf4cdf045b16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/bef1136972714eac852aaf4cdf045b16 2024-11-15T17:40:39,442 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d2c90382fd7e4549afde67e7739eabdf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/d2c90382fd7e4549afde67e7739eabdf 2024-11-15T17:40:39,443 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/dd22c5923b124a2cbfb5ccaad36ffc75 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/dd22c5923b124a2cbfb5ccaad36ffc75 2024-11-15T17:40:39,445 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ec486dbdf39541d9b99486ab746dab12 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/A/ec486dbdf39541d9b99486ab746dab12 2024-11-15T17:40:39,447 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/1fb9ad2eb513469283f11a849a7e7778 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/1fb9ad2eb513469283f11a849a7e7778 2024-11-15T17:40:39,449 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/213372609d89489d8a74458e33d9c5fa to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/213372609d89489d8a74458e33d9c5fa 2024-11-15T17:40:39,451 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/992435ef56f44231828e2ab15b81a2f6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/992435ef56f44231828e2ab15b81a2f6 2024-11-15T17:40:39,452 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/ec195e37de90420c8d1493dc6aa4dd56 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/B/ec195e37de90420c8d1493dc6aa4dd56 2024-11-15T17:40:39,455 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/19b13ed78ee14b838d518e9e84bbf9aa to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/19b13ed78ee14b838d518e9e84bbf9aa 2024-11-15T17:40:39,456 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/610be4d6f26f499ea2a9e1e6e9f1859e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/610be4d6f26f499ea2a9e1e6e9f1859e 2024-11-15T17:40:39,457 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/cf4b2862393541e48d8fed41de9e9f47 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/cf4b2862393541e48d8fed41de9e9f47 2024-11-15T17:40:39,459 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/db5c019143444d009f6b274774a3245a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/C/db5c019143444d009f6b274774a3245a 2024-11-15T17:40:39,462 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/recovered.edits/445.seqid to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23/recovered.edits/445.seqid 2024-11-15T17:40:39,463 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:40:39,464 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:40:39,465 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T17:40:39,469 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150c583b408d2f4b7594e0d396bca43b9f_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150c583b408d2f4b7594e0d396bca43b9f_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,471 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115153dd6623276445a918fadcab165e497_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115153dd6623276445a918fadcab165e497_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,472 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111527bce343f2c74c71ac3622b4d5e9f185_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111527bce343f2c74c71ac3622b4d5e9f185_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,473 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111529696087daf54b8a9c0fea63ccdaf513_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111529696087daf54b8a9c0fea63ccdaf513_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,474 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111540c8e28f3ab5424fbfa8c72886f1e2ba_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111540c8e28f3ab5424fbfa8c72886f1e2ba_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,476 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115523b4ad668cf4606ac54330d8afeb539_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115523b4ad668cf4606ac54330d8afeb539_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,478 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155311bd80fa5a473ca9b6fb8bebc4c22c_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155311bd80fa5a473ca9b6fb8bebc4c22c_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,479 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111554bae4e2d0dd4c69b8a081abeae8edc6_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111554bae4e2d0dd4c69b8a081abeae8edc6_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,481 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155d6ad2bcc07c49b6abea06342a093bea_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155d6ad2bcc07c49b6abea06342a093bea_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,483 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111566d93633f5d9419bbcc2cea74bab9047_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111566d93633f5d9419bbcc2cea74bab9047_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,484 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157b10131348db4308bbf6bb10e4b6c628_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157b10131348db4308bbf6bb10e4b6c628_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,486 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115811304d32f2c43b9b793caa7b9149e9d_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115811304d32f2c43b9b793caa7b9149e9d_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,487 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115849cb1ea2373434aade7405ec0d49bdb_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115849cb1ea2373434aade7405ec0d49bdb_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,489 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e387fd41f004a709871f9f537307837_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e387fd41f004a709871f9f537307837_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,491 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159edb33f479e74edf873871a8d102ad68_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159edb33f479e74edf873871a8d102ad68_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,492 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b16caad8d40c4ff5839d316b8b623c11_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b16caad8d40c4ff5839d316b8b623c11_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,494 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b210e1b04f164c668744c0ba80c4aeda_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b210e1b04f164c668744c0ba80c4aeda_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,495 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b9385ce0024c4a5586ed20581d4a1bb0_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b9385ce0024c4a5586ed20581d4a1bb0_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,497 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c990eee45c4f426092ae477cacb6a11a_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c990eee45c4f426092ae477cacb6a11a_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,498 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c9bceaeb4ef3421090bc03d46f7f4234_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c9bceaeb4ef3421090bc03d46f7f4234_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,500 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d53dbc365b274c059d84001b1d786c48_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d53dbc365b274c059d84001b1d786c48_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,501 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115daefbb70ab7f4f3abfbbedbcef04ae73_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115daefbb70ab7f4f3abfbbedbcef04ae73_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,503 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f615c6cdaac741d284bd5e2eacea129f_103dbae58ceff9ab6b93f5c0fda56a23 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f615c6cdaac741d284bd5e2eacea129f_103dbae58ceff9ab6b93f5c0fda56a23 2024-11-15T17:40:39,504 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:40:39,506 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:39,509 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:40:39,512 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:40:39,513 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:39,513 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:40:39,513 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731692439513"}]},"ts":"9223372036854775807"} 2024-11-15T17:40:39,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:40:39,516 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 103dbae58ceff9ab6b93f5c0fda56a23, NAME => 'TestAcidGuarantees,,1731692413239.103dbae58ceff9ab6b93f5c0fda56a23.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:40:39,516 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:40:39,516 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731692439516"}]},"ts":"9223372036854775807"} 2024-11-15T17:40:39,519 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:40:39,519 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:39,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-15T17:40:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T17:40:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:40:39,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:39,544 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=247 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1795595999_22 at /127.0.0.1:60554 [Waiting for operation #537] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1230616815_22 at /127.0.0.1:51400 [Waiting for operation #759] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1230616815_22 at /127.0.0.1:60542 [Waiting for operation #539] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1795595999_22 at /127.0.0.1:36546 [Waiting for operation #862] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=532 (was 392) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9460 (was 10075) 2024-11-15T17:40:39,552 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=247, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=532, ProcessCount=11, AvailableMemoryMB=9458 2024-11-15T17:40:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:40:39,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:40:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T17:40:39,556 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:40:39,556 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:39,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-15T17:40:39,557 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:40:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T17:40:39,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742065_1241 (size=959) 2024-11-15T17:40:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T17:40:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T17:40:39,964 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:40:39,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742066_1242 (size=53) 2024-11-15T17:40:39,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:40:39,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 96ad00ac4dbe2b71b39f7138f6d0e503, disabling compactions & flushes 2024-11-15T17:40:39,972 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:39,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:39,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. after waiting 0 ms 2024-11-15T17:40:39,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:39,972 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:39,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96ad00ac4dbe2b71b39f7138f6d0e503: Waiting for close lock at 1731692439972Disabling compacts and flushes for region at 1731692439972Disabling writes for close at 1731692439972Writing region close event to WAL at 1731692439972Closed at 1731692439972 2024-11-15T17:40:39,973 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:40:39,973 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731692439973"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692439973"}]},"ts":"1731692439973"} 2024-11-15T17:40:39,975 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:40:39,976 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:40:39,977 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692439976"}]},"ts":"1731692439976"} 2024-11-15T17:40:39,978 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T17:40:39,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, ASSIGN}] 2024-11-15T17:40:39,980 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, ASSIGN 2024-11-15T17:40:39,981 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, ASSIGN; state=OFFLINE, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=false 2024-11-15T17:40:40,131 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=96ad00ac4dbe2b71b39f7138f6d0e503, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, ASSIGN because future has completed 2024-11-15T17:40:40,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:40:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T17:40:40,289 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:40,289 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:40:40,289 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,289 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:40:40,289 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,290 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,291 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,292 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96ad00ac4dbe2b71b39f7138f6d0e503 columnFamilyName A 2024-11-15T17:40:40,292 DEBUG [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:40,293 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] regionserver.HStore(327): Store=96ad00ac4dbe2b71b39f7138f6d0e503/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:40,293 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,294 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96ad00ac4dbe2b71b39f7138f6d0e503 columnFamilyName B 2024-11-15T17:40:40,294 DEBUG [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:40,295 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] regionserver.HStore(327): Store=96ad00ac4dbe2b71b39f7138f6d0e503/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:40,295 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,296 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96ad00ac4dbe2b71b39f7138f6d0e503 columnFamilyName C 2024-11-15T17:40:40,296 DEBUG [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:40,297 INFO [StoreOpener-96ad00ac4dbe2b71b39f7138f6d0e503-1 {}] regionserver.HStore(327): Store=96ad00ac4dbe2b71b39f7138f6d0e503/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:40,297 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,300 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,301 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,302 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,302 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,302 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:40:40,304 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,305 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:40:40,306 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 96ad00ac4dbe2b71b39f7138f6d0e503; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61757856, jitterRate=-0.07973623275756836}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:40:40,306 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,307 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 96ad00ac4dbe2b71b39f7138f6d0e503: Running coprocessor pre-open hook at 1731692440290Writing region info on filesystem at 1731692440290Initializing all the Stores at 1731692440290Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692440291 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692440291Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692440291Cleaning up temporary data from old regions at 1731692440302 (+11 ms)Running coprocessor post-open hooks at 1731692440306 (+4 ms)Region opened successfully at 1731692440307 (+1 ms) 2024-11-15T17:40:40,308 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., pid=63, masterSystemTime=1731692440285 2024-11-15T17:40:40,310 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:40,310 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:40,310 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=96ad00ac4dbe2b71b39f7138f6d0e503, regionState=OPEN, openSeqNum=2, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:40:40,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T17:40:40,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 in 179 msec 2024-11-15T17:40:40,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-15T17:40:40,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, ASSIGN in 335 msec 2024-11-15T17:40:40,318 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:40:40,319 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692440318"}]},"ts":"1731692440318"} 2024-11-15T17:40:40,320 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T17:40:40,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:40:40,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 767 msec 2024-11-15T17:40:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T17:40:40,696 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:40,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30849442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b89852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,702 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:40:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 4e3b17656000,41205,1731692382118 2024-11-15T17:40:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@588d8843 2024-11-15T17:40:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:40:40,707 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:40:40,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,709 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e470870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,710 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3976ac96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,712 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708739dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,713 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753ff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202903d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,716 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e3586f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a69abfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,718 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32389ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129819a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,721 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3380d89e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4596b96c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5692157b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66db747f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,727 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61373c7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,729 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111aa29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,730 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b5cc04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,731 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bebcf39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,732 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eae796a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:40:40,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:40:40,735 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:40:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:40:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:40:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635e1516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:40:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:40:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:40,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:40:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c298b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:40:40,738 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,738 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,738 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,738 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:40,738 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,739 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,739 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,739 DEBUG [Thread-1136 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,739 DEBUG [Thread-1136 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,739 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,739 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-15T17:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,740 DEBUG [Thread-1127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:40:40,740 DEBUG [Thread-1133 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,741 DEBUG [Thread-1133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T17:40:40,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:40:40,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:40:40,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,742 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:40,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,743 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:40,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:40:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:40:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:40,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:40,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:40:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692500778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692500778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692500781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692500779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692500782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ae0d7712e7fa4da99c0b0057ed682ef4 is 50, key is test_row_0/A:col10/1731692440750/Put/seqid=0 2024-11-15T17:40:40,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742067_1243 (size=12001) 2024-11-15T17:40:40,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ae0d7712e7fa4da99c0b0057ed682ef4 2024-11-15T17:40:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T17:40:40,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/3b8d16bf68104e03b8133ae8ced9ad5e is 50, key is test_row_0/B:col10/1731692440750/Put/seqid=0 2024-11-15T17:40:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742068_1244 (size=12001) 2024-11-15T17:40:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692500885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692500885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T17:40:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:40,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:40,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:40,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692500910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692500911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:40,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692500911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T17:40:41,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T17:40:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:41,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:41,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692501095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692501095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692501126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692501126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692501126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T17:40:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:41,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:41,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:40:41,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/3b8d16bf68104e03b8133ae8ced9ad5e 2024-11-15T17:40:41,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/2580a64296c54d8186149fcb4ae5fecf is 50, key is test_row_0/C:col10/1731692440750/Put/seqid=0 2024-11-15T17:40:41,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742069_1245 (size=12001) 2024-11-15T17:40:41,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/2580a64296c54d8186149fcb4ae5fecf 2024-11-15T17:40:41,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ae0d7712e7fa4da99c0b0057ed682ef4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ae0d7712e7fa4da99c0b0057ed682ef4 2024-11-15T17:40:41,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ae0d7712e7fa4da99c0b0057ed682ef4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T17:40:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/3b8d16bf68104e03b8133ae8ced9ad5e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/3b8d16bf68104e03b8133ae8ced9ad5e 2024-11-15T17:40:41,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/3b8d16bf68104e03b8133ae8ced9ad5e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T17:40:41,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/2580a64296c54d8186149fcb4ae5fecf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2580a64296c54d8186149fcb4ae5fecf 2024-11-15T17:40:41,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2580a64296c54d8186149fcb4ae5fecf, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T17:40:41,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 598ms, sequenceid=14, compaction requested=false 2024-11-15T17:40:41,349 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T17:40:41,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T17:40:41,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T17:40:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:41,372 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:40:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7782eab5d71a4ae783486b8dc88f64f0 is 50, key is test_row_0/A:col10/1731692440769/Put/seqid=0 2024-11-15T17:40:41,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742070_1246 (size=12001) 2024-11-15T17:40:41,382 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7782eab5d71a4ae783486b8dc88f64f0 2024-11-15T17:40:41,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/45a9a98e86a24719a36ea5236fc5eb52 is 50, key is test_row_0/B:col10/1731692440769/Put/seqid=0 2024-11-15T17:40:41,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742071_1247 (size=12001) 2024-11-15T17:40:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:41,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:41,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692501427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692501428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692501435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692501436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692501436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692501535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692501536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692501745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692501745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,793 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/45a9a98e86a24719a36ea5236fc5eb52 2024-11-15T17:40:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/7367b4e79b054a60b4fc240783d3636a is 50, key is test_row_0/C:col10/1731692440769/Put/seqid=0 2024-11-15T17:40:41,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742072_1248 (size=12001) 2024-11-15T17:40:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T17:40:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692501945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692501945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692501945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692502055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692502055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,204 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/7367b4e79b054a60b4fc240783d3636a 2024-11-15T17:40:42,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7782eab5d71a4ae783486b8dc88f64f0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7782eab5d71a4ae783486b8dc88f64f0 2024-11-15T17:40:42,213 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7782eab5d71a4ae783486b8dc88f64f0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T17:40:42,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/45a9a98e86a24719a36ea5236fc5eb52 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/45a9a98e86a24719a36ea5236fc5eb52 2024-11-15T17:40:42,217 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/45a9a98e86a24719a36ea5236fc5eb52, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T17:40:42,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/7367b4e79b054a60b4fc240783d3636a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7367b4e79b054a60b4fc240783d3636a 2024-11-15T17:40:42,222 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7367b4e79b054a60b4fc240783d3636a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T17:40:42,223 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 851ms, sequenceid=37, compaction requested=false 2024-11-15T17:40:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:42,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-15T17:40:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-15T17:40:42,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T17:40:42,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4810 sec 2024-11-15T17:40:42,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4880 sec 2024-11-15T17:40:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:42,568 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:42,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:40:42,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e is 50, key is test_row_0/A:col10/1731692442568/Put/seqid=0 2024-11-15T17:40:42,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742073_1249 (size=14341) 2024-11-15T17:40:42,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692502600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692502603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692502715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692502715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T17:40:42,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:42,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T17:40:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T17:40:42,898 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:42,898 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:42,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:42,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692502925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692502925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692502956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692502956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,959 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:42,959 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:42,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692502965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:42,968 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:42,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e 2024-11-15T17:40:42,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/446ac77853be4387b0c5567059a669da is 50, key is test_row_0/B:col10/1731692442568/Put/seqid=0 2024-11-15T17:40:42,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742074_1250 (size=12001) 2024-11-15T17:40:42,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/446ac77853be4387b0c5567059a669da 2024-11-15T17:40:42,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/578f22cd946f4ea4893f006fd095cbac is 50, key is test_row_0/C:col10/1731692442568/Put/seqid=0 2024-11-15T17:40:42,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742075_1251 (size=12001) 2024-11-15T17:40:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T17:40:43,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T17:40:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T17:40:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T17:40:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692503236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:43,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692503236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:43,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T17:40:43,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:43,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:43,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/578f22cd946f4ea4893f006fd095cbac 2024-11-15T17:40:43,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e 2024-11-15T17:40:43,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e, entries=200, sequenceid=52, filesize=14.0 K 2024-11-15T17:40:43,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/446ac77853be4387b0c5567059a669da as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/446ac77853be4387b0c5567059a669da 2024-11-15T17:40:43,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/446ac77853be4387b0c5567059a669da, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T17:40:43,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/578f22cd946f4ea4893f006fd095cbac as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/578f22cd946f4ea4893f006fd095cbac 2024-11-15T17:40:43,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/578f22cd946f4ea4893f006fd095cbac, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T17:40:43,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 849ms, sequenceid=52, compaction requested=true 2024-11-15T17:40:43,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:43,418 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:43,418 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:43,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:43,418 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:43,418 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:43,419 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:43,419 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:43,419 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,419 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,419 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ae0d7712e7fa4da99c0b0057ed682ef4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7782eab5d71a4ae783486b8dc88f64f0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=37.4 K 2024-11-15T17:40:43,419 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/3b8d16bf68104e03b8133ae8ced9ad5e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/45a9a98e86a24719a36ea5236fc5eb52, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/446ac77853be4387b0c5567059a669da] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=35.2 K 2024-11-15T17:40:43,419 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0d7712e7fa4da99c0b0057ed682ef4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731692440750 2024-11-15T17:40:43,419 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8d16bf68104e03b8133ae8ced9ad5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731692440750 2024-11-15T17:40:43,419 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7782eab5d71a4ae783486b8dc88f64f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731692440769 2024-11-15T17:40:43,419 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45a9a98e86a24719a36ea5236fc5eb52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731692440769 2024-11-15T17:40:43,420 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fdcb3fd0f61d41b0a6fc9f8c0d8ada2e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692441424 2024-11-15T17:40:43,420 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 446ac77853be4387b0c5567059a669da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692441424 2024-11-15T17:40:43,428 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:43,428 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/565e9f33cc974591881e78cac3ac30de is 50, key is test_row_0/A:col10/1731692442568/Put/seqid=0 2024-11-15T17:40:43,429 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#203 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:43,429 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/5d1ebf285d4246fa83982dc597698e11 is 50, key is test_row_0/B:col10/1731692442568/Put/seqid=0 2024-11-15T17:40:43,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742077_1253 (size=12104) 2024-11-15T17:40:43,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742076_1252 (size=12104) 2024-11-15T17:40:43,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T17:40:43,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,510 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T17:40:43,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/408a3174674a4eb1bda0fc2cfda20733 is 50, key is test_row_0/A:col10/1731692442601/Put/seqid=0 2024-11-15T17:40:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742078_1254 (size=12001) 2024-11-15T17:40:43,522 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/408a3174674a4eb1bda0fc2cfda20733 2024-11-15T17:40:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T17:40:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/34184ebdc5554a24af8fdf744e41136b is 50, key is test_row_0/B:col10/1731692442601/Put/seqid=0 2024-11-15T17:40:43,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742079_1255 (size=12001) 2024-11-15T17:40:43,550 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/34184ebdc5554a24af8fdf744e41136b 2024-11-15T17:40:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/5e7b174fc1564a8ca1fb642253ad0594 is 50, key is test_row_0/C:col10/1731692442601/Put/seqid=0 2024-11-15T17:40:43,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742080_1256 (size=12001) 2024-11-15T17:40:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692503770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:43,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692503775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:43,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:43,871 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/5d1ebf285d4246fa83982dc597698e11 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/5d1ebf285d4246fa83982dc597698e11 2024-11-15T17:40:43,875 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into 5d1ebf285d4246fa83982dc597698e11(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:43,875 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:43,875 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=13, startTime=1731692443418; duration=0sec 2024-11-15T17:40:43,876 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:43,876 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:43,876 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:43,877 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:43,877 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:43,877 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:43,877 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2580a64296c54d8186149fcb4ae5fecf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7367b4e79b054a60b4fc240783d3636a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/578f22cd946f4ea4893f006fd095cbac] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=35.2 K 2024-11-15T17:40:43,877 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2580a64296c54d8186149fcb4ae5fecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731692440750 2024-11-15T17:40:43,878 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7367b4e79b054a60b4fc240783d3636a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731692440769 2024-11-15T17:40:43,879 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 578f22cd946f4ea4893f006fd095cbac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692441424 2024-11-15T17:40:43,884 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/565e9f33cc974591881e78cac3ac30de as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/565e9f33cc974591881e78cac3ac30de 2024-11-15T17:40:43,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692503886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:43,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692503886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:43,889 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:43,889 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/1f66670625794b07bbcbe012a26ba7a2 is 50, key is test_row_0/C:col10/1731692442568/Put/seqid=0 2024-11-15T17:40:43,892 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into 565e9f33cc974591881e78cac3ac30de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:43,892 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:43,893 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=13, startTime=1731692443417; duration=0sec 2024-11-15T17:40:43,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:43,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:43,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742081_1257 (size=12104) 2024-11-15T17:40:43,910 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/1f66670625794b07bbcbe012a26ba7a2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1f66670625794b07bbcbe012a26ba7a2 2024-11-15T17:40:43,914 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into 1f66670625794b07bbcbe012a26ba7a2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:43,914 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:43,914 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=13, startTime=1731692443418; duration=0sec 2024-11-15T17:40:43,915 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:43,915 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:43,961 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:40:43,994 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/5e7b174fc1564a8ca1fb642253ad0594 2024-11-15T17:40:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/408a3174674a4eb1bda0fc2cfda20733 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/408a3174674a4eb1bda0fc2cfda20733 2024-11-15T17:40:44,005 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/408a3174674a4eb1bda0fc2cfda20733, entries=150, sequenceid=73, filesize=11.7 K 2024-11-15T17:40:44,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/34184ebdc5554a24af8fdf744e41136b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/34184ebdc5554a24af8fdf744e41136b 2024-11-15T17:40:44,010 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/34184ebdc5554a24af8fdf744e41136b, entries=150, sequenceid=73, filesize=11.7 K 2024-11-15T17:40:44,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/5e7b174fc1564a8ca1fb642253ad0594 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/5e7b174fc1564a8ca1fb642253ad0594 2024-11-15T17:40:44,016 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/5e7b174fc1564a8ca1fb642253ad0594, entries=150, sequenceid=73, filesize=11.7 K 2024-11-15T17:40:44,017 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 507ms, sequenceid=73, compaction requested=false 2024-11-15T17:40:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T17:40:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T17:40:44,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T17:40:44,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1210 sec 2024-11-15T17:40:44,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1250 sec 2024-11-15T17:40:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T17:40:44,036 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:44,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T17:40:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T17:40:44,040 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:44,040 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:44,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:44,098 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:44,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:40:44,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/50ee1d55c1c84acb918fe7ab22bd6536 is 50, key is test_row_0/A:col10/1731692443769/Put/seqid=0 2024-11-15T17:40:44,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742082_1258 (size=16681) 2024-11-15T17:40:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692504134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692504135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T17:40:44,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692504246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692504246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:44,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:44,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T17:40:44,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692504456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692504456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:44,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:44,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/50ee1d55c1c84acb918fe7ab22bd6536 2024-11-15T17:40:44,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/92e4ad3dfa08435397383edb2984e035 is 50, key is test_row_0/B:col10/1731692443769/Put/seqid=0 2024-11-15T17:40:44,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742083_1259 (size=12001) 2024-11-15T17:40:44,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:44,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:44,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T17:40:44,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692504765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692504765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:44,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:44,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:44,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:44,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692504966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,968 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692504976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692504976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:44,979 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,980 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:44,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/92e4ad3dfa08435397383edb2984e035 2024-11-15T17:40:45,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/d67c0498dab14fc4989cbdd8f49369c3 is 50, key is test_row_0/C:col10/1731692443769/Put/seqid=0 2024-11-15T17:40:45,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742084_1260 (size=12001) 2024-11-15T17:40:45,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:45,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:45,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T17:40:45,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692505275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692505275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:45,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:45,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/d67c0498dab14fc4989cbdd8f49369c3 2024-11-15T17:40:45,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:45,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:45,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/50ee1d55c1c84acb918fe7ab22bd6536 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/50ee1d55c1c84acb918fe7ab22bd6536 2024-11-15T17:40:45,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/50ee1d55c1c84acb918fe7ab22bd6536, entries=250, sequenceid=93, filesize=16.3 K 2024-11-15T17:40:45,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/92e4ad3dfa08435397383edb2984e035 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/92e4ad3dfa08435397383edb2984e035 2024-11-15T17:40:45,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/92e4ad3dfa08435397383edb2984e035, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T17:40:45,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/d67c0498dab14fc4989cbdd8f49369c3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/d67c0498dab14fc4989cbdd8f49369c3 2024-11-15T17:40:45,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/d67c0498dab14fc4989cbdd8f49369c3, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T17:40:45,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1348ms, sequenceid=93, compaction requested=true 2024-11-15T17:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:45,447 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:45,447 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:45,448 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:45,448 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:45,448 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,448 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/565e9f33cc974591881e78cac3ac30de, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/408a3174674a4eb1bda0fc2cfda20733, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/50ee1d55c1c84acb918fe7ab22bd6536] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=39.8 K 2024-11-15T17:40:45,449 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:45,449 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:45,449 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,449 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/5d1ebf285d4246fa83982dc597698e11, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/34184ebdc5554a24af8fdf744e41136b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/92e4ad3dfa08435397383edb2984e035] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=35.3 K 2024-11-15T17:40:45,451 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1ebf285d4246fa83982dc597698e11, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692441424 2024-11-15T17:40:45,451 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 565e9f33cc974591881e78cac3ac30de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692441424 2024-11-15T17:40:45,452 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408a3174674a4eb1bda0fc2cfda20733, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731692442597 2024-11-15T17:40:45,452 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 34184ebdc5554a24af8fdf744e41136b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731692442597 2024-11-15T17:40:45,453 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50ee1d55c1c84acb918fe7ab22bd6536, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731692443751 2024-11-15T17:40:45,453 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e4ad3dfa08435397383edb2984e035, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731692443769 2024-11-15T17:40:45,460 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:45,460 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/25ab2f3000184da68ad46ba77e610b20 is 50, key is test_row_0/B:col10/1731692443769/Put/seqid=0 2024-11-15T17:40:45,462 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:45,463 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/9958d727425f4ad7a392f8379e7a4151 is 50, key is test_row_0/A:col10/1731692443769/Put/seqid=0 2024-11-15T17:40:45,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742085_1261 (size=12207) 2024-11-15T17:40:45,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742086_1262 (size=12207) 2024-11-15T17:40:45,483 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/25ab2f3000184da68ad46ba77e610b20 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/25ab2f3000184da68ad46ba77e610b20 2024-11-15T17:40:45,489 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into 25ab2f3000184da68ad46ba77e610b20(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:45,489 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:45,489 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=13, startTime=1731692445447; duration=0sec 2024-11-15T17:40:45,489 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:45,489 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:45,489 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:45,490 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:45,490 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:45,490 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,490 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1f66670625794b07bbcbe012a26ba7a2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/5e7b174fc1564a8ca1fb642253ad0594, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/d67c0498dab14fc4989cbdd8f49369c3] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=35.3 K 2024-11-15T17:40:45,491 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f66670625794b07bbcbe012a26ba7a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731692441424 2024-11-15T17:40:45,491 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e7b174fc1564a8ca1fb642253ad0594, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731692442597 2024-11-15T17:40:45,491 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d67c0498dab14fc4989cbdd8f49369c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731692443769 2024-11-15T17:40:45,499 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:45,500 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/f8005dafeee34be3b6b03ce7548e1202 is 50, key is test_row_0/C:col10/1731692443769/Put/seqid=0 2024-11-15T17:40:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742087_1263 (size=12207) 2024-11-15T17:40:45,546 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/f8005dafeee34be3b6b03ce7548e1202 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/f8005dafeee34be3b6b03ce7548e1202 2024-11-15T17:40:45,552 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into f8005dafeee34be3b6b03ce7548e1202(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:45,552 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:45,552 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=13, startTime=1731692445447; duration=0sec 2024-11-15T17:40:45,552 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:45,554 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:45,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T17:40:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:45,577 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T17:40:45,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/c7427b94dfd8464b995a84eca181de85 is 50, key is test_row_0/A:col10/1731692444134/Put/seqid=0 2024-11-15T17:40:45,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742088_1264 (size=12001) 2024-11-15T17:40:45,601 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/c7427b94dfd8464b995a84eca181de85 2024-11-15T17:40:45,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/777dffe8e59a4714a3ff1d8bd90639fe is 50, key is test_row_0/B:col10/1731692444134/Put/seqid=0 2024-11-15T17:40:45,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742089_1265 (size=12001) 2024-11-15T17:40:45,884 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/9958d727425f4ad7a392f8379e7a4151 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9958d727425f4ad7a392f8379e7a4151 2024-11-15T17:40:45,890 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into 9958d727425f4ad7a392f8379e7a4151(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:45,890 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:45,890 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=13, startTime=1731692445447; duration=0sec 2024-11-15T17:40:45,890 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:45,890 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:46,058 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/777dffe8e59a4714a3ff1d8bd90639fe 2024-11-15T17:40:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/14a0c768864346b99a3725dc6f7691cb is 50, key is test_row_0/C:col10/1731692444134/Put/seqid=0 2024-11-15T17:40:46,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742090_1266 (size=12001) 2024-11-15T17:40:46,114 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/14a0c768864346b99a3725dc6f7691cb 2024-11-15T17:40:46,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/c7427b94dfd8464b995a84eca181de85 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c7427b94dfd8464b995a84eca181de85 2024-11-15T17:40:46,126 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c7427b94dfd8464b995a84eca181de85, entries=150, sequenceid=114, filesize=11.7 K 2024-11-15T17:40:46,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/777dffe8e59a4714a3ff1d8bd90639fe as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/777dffe8e59a4714a3ff1d8bd90639fe 2024-11-15T17:40:46,133 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/777dffe8e59a4714a3ff1d8bd90639fe, entries=150, sequenceid=114, filesize=11.7 K 2024-11-15T17:40:46,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/14a0c768864346b99a3725dc6f7691cb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/14a0c768864346b99a3725dc6f7691cb 2024-11-15T17:40:46,145 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/14a0c768864346b99a3725dc6f7691cb, entries=150, sequenceid=114, filesize=11.7 K 2024-11-15T17:40:46,147 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 570ms, sequenceid=114, compaction requested=false 2024-11-15T17:40:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T17:40:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T17:40:46,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T17:40:46,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1100 sec 2024-11-15T17:40:46,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.1150 sec 2024-11-15T17:40:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T17:40:46,186 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:46,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T17:40:46,191 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T17:40:46,192 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:46,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T17:40:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:46,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:46,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:40:46,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/caaf19b2cbe846d0902325d01922a605 is 50, key is test_row_0/A:col10/1731692446301/Put/seqid=0 2024-11-15T17:40:46,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742091_1267 (size=14341) 2024-11-15T17:40:46,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/caaf19b2cbe846d0902325d01922a605 2024-11-15T17:40:46,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/fd5a2e748fd74c118be8606d2dcb93e4 is 50, key is test_row_0/B:col10/1731692446301/Put/seqid=0 2024-11-15T17:40:46,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:46,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:46,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742092_1268 (size=12001) 2024-11-15T17:40:46,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/fd5a2e748fd74c118be8606d2dcb93e4 2024-11-15T17:40:46,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/b69f3bd089a3489b9bdbad1e068799f1 is 50, key is test_row_0/C:col10/1731692446301/Put/seqid=0 2024-11-15T17:40:46,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742093_1269 (size=12001) 2024-11-15T17:40:46,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/b69f3bd089a3489b9bdbad1e068799f1 2024-11-15T17:40:46,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692506377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692506379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/caaf19b2cbe846d0902325d01922a605 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/caaf19b2cbe846d0902325d01922a605 2024-11-15T17:40:46,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/caaf19b2cbe846d0902325d01922a605, entries=200, sequenceid=126, filesize=14.0 K 2024-11-15T17:40:46,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/fd5a2e748fd74c118be8606d2dcb93e4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fd5a2e748fd74c118be8606d2dcb93e4 2024-11-15T17:40:46,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fd5a2e748fd74c118be8606d2dcb93e4, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T17:40:46,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/b69f3bd089a3489b9bdbad1e068799f1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b69f3bd089a3489b9bdbad1e068799f1 2024-11-15T17:40:46,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b69f3bd089a3489b9bdbad1e068799f1, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T17:40:46,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 114ms, sequenceid=126, compaction requested=true 2024-11-15T17:40:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:46,416 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:46,416 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:46,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:46,417 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:46,418 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:46,418 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,418 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/25ab2f3000184da68ad46ba77e610b20, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/777dffe8e59a4714a3ff1d8bd90639fe, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fd5a2e748fd74c118be8606d2dcb93e4] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=35.4 K 2024-11-15T17:40:46,418 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:46,418 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:46,418 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,418 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9958d727425f4ad7a392f8379e7a4151, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c7427b94dfd8464b995a84eca181de85, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/caaf19b2cbe846d0902325d01922a605] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=37.6 K 2024-11-15T17:40:46,418 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ab2f3000184da68ad46ba77e610b20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731692443769 2024-11-15T17:40:46,419 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9958d727425f4ad7a392f8379e7a4151, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731692443769 2024-11-15T17:40:46,420 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7427b94dfd8464b995a84eca181de85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731692444128 2024-11-15T17:40:46,420 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 777dffe8e59a4714a3ff1d8bd90639fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731692444128 2024-11-15T17:40:46,420 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting caaf19b2cbe846d0902325d01922a605, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731692446292 2024-11-15T17:40:46,420 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5a2e748fd74c118be8606d2dcb93e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731692446292 2024-11-15T17:40:46,429 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:46,429 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:46,430 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/f3d9051d0a3e4a16a76660c1bf628719 is 50, key is test_row_0/B:col10/1731692446301/Put/seqid=0 2024-11-15T17:40:46,430 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/58c780906b1846d783f5cecdbd842d8f is 50, key is test_row_0/A:col10/1731692446301/Put/seqid=0 2024-11-15T17:40:46,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742094_1270 (size=12309) 2024-11-15T17:40:46,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742095_1271 (size=12309) 2024-11-15T17:40:46,486 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/58c780906b1846d783f5cecdbd842d8f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/58c780906b1846d783f5cecdbd842d8f 2024-11-15T17:40:46,494 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into 58c780906b1846d783f5cecdbd842d8f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:46,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:46,494 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=13, startTime=1731692446416; duration=0sec 2024-11-15T17:40:46,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:46,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:46,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:46,496 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:46,496 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:46,496 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,496 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/f8005dafeee34be3b6b03ce7548e1202, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/14a0c768864346b99a3725dc6f7691cb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b69f3bd089a3489b9bdbad1e068799f1] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=35.4 K 2024-11-15T17:40:46,497 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8005dafeee34be3b6b03ce7548e1202, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731692443769 2024-11-15T17:40:46,497 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a0c768864346b99a3725dc6f7691cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731692444128 2024-11-15T17:40:46,497 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting b69f3bd089a3489b9bdbad1e068799f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731692446292 2024-11-15T17:40:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:46,500 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:46,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T17:40:46,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T17:40:46,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/024be5f1469c463fa6fdb4c250398490 is 50, key is test_row_0/A:col10/1731692446498/Put/seqid=0 2024-11-15T17:40:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692506519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692506519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,529 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:46,529 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/2b3b7ab89fe44d3eb7a734218d29ded7 is 50, key is test_row_0/C:col10/1731692446301/Put/seqid=0 2024-11-15T17:40:46,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742096_1272 (size=12151) 2024-11-15T17:40:46,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742097_1273 (size=12309) 2024-11-15T17:40:46,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692506625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692506625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:46,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:46,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T17:40:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692506835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692506835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:46,880 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/f3d9051d0a3e4a16a76660c1bf628719 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/f3d9051d0a3e4a16a76660c1bf628719 2024-11-15T17:40:46,886 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into f3d9051d0a3e4a16a76660c1bf628719(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:46,886 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:46,886 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=13, startTime=1731692446416; duration=0sec 2024-11-15T17:40:46,886 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:46,886 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:46,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:46,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/024be5f1469c463fa6fdb4c250398490 2024-11-15T17:40:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:46,980 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/2b3b7ab89fe44d3eb7a734218d29ded7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2b3b7ab89fe44d3eb7a734218d29ded7 2024-11-15T17:40:46,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/842330119ae64594bc8b0a9f6bcf4a93 is 50, key is test_row_0/B:col10/1731692446498/Put/seqid=0 2024-11-15T17:40:46,987 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into 2b3b7ab89fe44d3eb7a734218d29ded7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:46,988 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:46,988 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=13, startTime=1731692446416; duration=0sec 2024-11-15T17:40:46,988 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:46,988 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:47,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742098_1274 (size=12151) 2024-11-15T17:40:47,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:47,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:47,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692507146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692507146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:47,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:47,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:47,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T17:40:47,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/842330119ae64594bc8b0a9f6bcf4a93 2024-11-15T17:40:47,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/440e2c7a735646298117d64458967812 is 50, key is test_row_0/C:col10/1731692446498/Put/seqid=0 2024-11-15T17:40:47,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:47,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:47,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:47,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742099_1275 (size=12151) 2024-11-15T17:40:47,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/440e2c7a735646298117d64458967812 2024-11-15T17:40:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/024be5f1469c463fa6fdb4c250398490 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/024be5f1469c463fa6fdb4c250398490 2024-11-15T17:40:47,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/024be5f1469c463fa6fdb4c250398490, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T17:40:47,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/842330119ae64594bc8b0a9f6bcf4a93 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/842330119ae64594bc8b0a9f6bcf4a93 2024-11-15T17:40:47,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/842330119ae64594bc8b0a9f6bcf4a93, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T17:40:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/440e2c7a735646298117d64458967812 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/440e2c7a735646298117d64458967812 2024-11-15T17:40:47,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/440e2c7a735646298117d64458967812, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T17:40:47,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1009ms, sequenceid=154, compaction requested=false 2024-11-15T17:40:47,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:47,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T17:40:47,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,598 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T17:40:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/9c9ae0ccca404f7b819c730fda651bfa is 50, key is test_row_0/A:col10/1731692446505/Put/seqid=0 2024-11-15T17:40:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742100_1276 (size=12151) 2024-11-15T17:40:47,655 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/9c9ae0ccca404f7b819c730fda651bfa 2024-11-15T17:40:47,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/09263af7edd4457bb567fc0141f29b4e is 50, key is test_row_0/B:col10/1731692446505/Put/seqid=0 2024-11-15T17:40:47,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742101_1277 (size=12151) 2024-11-15T17:40:47,733 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/09263af7edd4457bb567fc0141f29b4e 2024-11-15T17:40:47,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/b6b31686cc614fa780d788003e17f7fa is 50, key is test_row_0/C:col10/1731692446505/Put/seqid=0 2024-11-15T17:40:47,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742102_1278 (size=12151) 2024-11-15T17:40:47,764 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/b6b31686cc614fa780d788003e17f7fa 2024-11-15T17:40:47,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/9c9ae0ccca404f7b819c730fda651bfa as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9c9ae0ccca404f7b819c730fda651bfa 2024-11-15T17:40:47,779 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9c9ae0ccca404f7b819c730fda651bfa, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:40:47,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/09263af7edd4457bb567fc0141f29b4e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/09263af7edd4457bb567fc0141f29b4e 2024-11-15T17:40:47,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692507782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:47,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692507788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:47,793 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/09263af7edd4457bb567fc0141f29b4e, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:40:47,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/b6b31686cc614fa780d788003e17f7fa as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b6b31686cc614fa780d788003e17f7fa 2024-11-15T17:40:47,799 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b6b31686cc614fa780d788003e17f7fa, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T17:40:47,799 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 201ms, sequenceid=165, compaction requested=true 2024-11-15T17:40:47,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:47,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:47,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T17:40:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T17:40:47,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T17:40:47,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-15T17:40:47,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6160 sec 2024-11-15T17:40:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:47,898 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:47,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T17:40:47,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/d7341c389fb345efa11c0756f06a9e91 is 50, key is test_row_0/A:col10/1731692447749/Put/seqid=0 2024-11-15T17:40:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692507917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692507917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:47,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742103_1279 (size=12151) 2024-11-15T17:40:48,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692508025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692508026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:48,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692508235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:48,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:48,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692508235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:48,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:48,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T17:40:48,335 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:48,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T17:40:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T17:40:48,341 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:48,342 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:48,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:48,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/d7341c389fb345efa11c0756f06a9e91 2024-11-15T17:40:48,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/6f18f4ec00e44a879a66398948ad299d is 50, key is test_row_0/B:col10/1731692447749/Put/seqid=0 2024-11-15T17:40:48,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742104_1280 (size=12151) 2024-11-15T17:40:48,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/6f18f4ec00e44a879a66398948ad299d 2024-11-15T17:40:48,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/70144e03173c4930a810f4ab6b8a627e is 50, key is test_row_0/C:col10/1731692447749/Put/seqid=0 2024-11-15T17:40:48,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742105_1281 (size=12151) 2024-11-15T17:40:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T17:40:48,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T17:40:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692508546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:48,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692508546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:48,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:48,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T17:40:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T17:40:48,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/70144e03173c4930a810f4ab6b8a627e 2024-11-15T17:40:48,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/d7341c389fb345efa11c0756f06a9e91 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/d7341c389fb345efa11c0756f06a9e91 2024-11-15T17:40:48,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/d7341c389fb345efa11c0756f06a9e91, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T17:40:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/6f18f4ec00e44a879a66398948ad299d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/6f18f4ec00e44a879a66398948ad299d 2024-11-15T17:40:48,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/6f18f4ec00e44a879a66398948ad299d, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T17:40:48,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/70144e03173c4930a810f4ab6b8a627e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/70144e03173c4930a810f4ab6b8a627e 2024-11-15T17:40:48,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T17:40:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:48,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/70144e03173c4930a810f4ab6b8a627e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T17:40:48,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 910ms, sequenceid=193, compaction requested=true 2024-11-15T17:40:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:48,809 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:48,809 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:48,815 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:48,815 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:48,815 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,815 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/f3d9051d0a3e4a16a76660c1bf628719, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/842330119ae64594bc8b0a9f6bcf4a93, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/09263af7edd4457bb567fc0141f29b4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/6f18f4ec00e44a879a66398948ad299d] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=47.6 K 2024-11-15T17:40:48,816 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:48,816 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:48,816 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f3d9051d0a3e4a16a76660c1bf628719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731692446292 2024-11-15T17:40:48,816 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,816 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/58c780906b1846d783f5cecdbd842d8f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/024be5f1469c463fa6fdb4c250398490, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9c9ae0ccca404f7b819c730fda651bfa, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/d7341c389fb345efa11c0756f06a9e91] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=47.6 K 2024-11-15T17:40:48,816 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 842330119ae64594bc8b0a9f6bcf4a93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731692446375 2024-11-15T17:40:48,816 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c780906b1846d783f5cecdbd842d8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731692446292 2024-11-15T17:40:48,816 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 09263af7edd4457bb567fc0141f29b4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731692446505 2024-11-15T17:40:48,816 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024be5f1469c463fa6fdb4c250398490, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731692446375 2024-11-15T17:40:48,817 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f18f4ec00e44a879a66398948ad299d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731692447749 2024-11-15T17:40:48,817 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c9ae0ccca404f7b819c730fda651bfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731692446505 2024-11-15T17:40:48,817 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7341c389fb345efa11c0756f06a9e91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731692447749 2024-11-15T17:40:48,828 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:48,829 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/a3018de4a32d4ee48834dd2c59a7ed50 is 50, key is test_row_0/B:col10/1731692447749/Put/seqid=0 2024-11-15T17:40:48,837 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:48,837 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/fde029ebe923486a86de43acff7cc52f is 50, key is test_row_0/A:col10/1731692447749/Put/seqid=0 2024-11-15T17:40:48,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742106_1282 (size=12595) 2024-11-15T17:40:48,845 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/a3018de4a32d4ee48834dd2c59a7ed50 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a3018de4a32d4ee48834dd2c59a7ed50 2024-11-15T17:40:48,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742107_1283 (size=12595) 2024-11-15T17:40:48,850 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/fde029ebe923486a86de43acff7cc52f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fde029ebe923486a86de43acff7cc52f 2024-11-15T17:40:48,850 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into a3018de4a32d4ee48834dd2c59a7ed50(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:48,851 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:48,851 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=12, startTime=1731692448809; duration=0sec 2024-11-15T17:40:48,851 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:48,851 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:48,851 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:48,864 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:48,864 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:48,864 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,864 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2b3b7ab89fe44d3eb7a734218d29ded7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/440e2c7a735646298117d64458967812, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b6b31686cc614fa780d788003e17f7fa, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/70144e03173c4930a810f4ab6b8a627e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=47.6 K 2024-11-15T17:40:48,865 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3b7ab89fe44d3eb7a734218d29ded7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731692446292 2024-11-15T17:40:48,865 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 440e2c7a735646298117d64458967812, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731692446375 2024-11-15T17:40:48,865 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b31686cc614fa780d788003e17f7fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731692446505 2024-11-15T17:40:48,866 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 70144e03173c4930a810f4ab6b8a627e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731692447749 2024-11-15T17:40:48,871 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into fde029ebe923486a86de43acff7cc52f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:48,871 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:48,871 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=12, startTime=1731692448809; duration=0sec 2024-11-15T17:40:48,871 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:48,871 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:48,881 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:48,882 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/c138e82d4fce4edea13db8ba213b2daf is 50, key is test_row_0/C:col10/1731692447749/Put/seqid=0 2024-11-15T17:40:48,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742108_1284 (size=12595) 2024-11-15T17:40:48,922 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/c138e82d4fce4edea13db8ba213b2daf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/c138e82d4fce4edea13db8ba213b2daf 2024-11-15T17:40:48,928 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into c138e82d4fce4edea13db8ba213b2daf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:48,928 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:48,929 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=12, startTime=1731692448809; duration=0sec 2024-11-15T17:40:48,929 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:48,929 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:48,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T17:40:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:48,956 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T17:40:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T17:40:48,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/0094df76678f4b3297d31f98af019a7e is 50, key is test_row_0/A:col10/1731692447914/Put/seqid=0 2024-11-15T17:40:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742109_1285 (size=12151) 2024-11-15T17:40:48,986 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/0094df76678f4b3297d31f98af019a7e 2024-11-15T17:40:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:48,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/d6505d0679dc4842b236077f700cfe33 is 50, key is test_row_0/B:col10/1731692447914/Put/seqid=0 2024-11-15T17:40:49,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742110_1286 (size=12151) 2024-11-15T17:40:49,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692509036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692509038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692509039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692509055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692509055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692509145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692509145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692509145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692509355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692509355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692509355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,408 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/d6505d0679dc4842b236077f700cfe33 2024-11-15T17:40:49,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/1b1b9913769b47fdb1ae2a36b2649d30 is 50, key is test_row_0/C:col10/1731692447914/Put/seqid=0 2024-11-15T17:40:49,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742111_1287 (size=12151) 2024-11-15T17:40:49,418 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/1b1b9913769b47fdb1ae2a36b2649d30 2024-11-15T17:40:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/0094df76678f4b3297d31f98af019a7e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0094df76678f4b3297d31f98af019a7e 2024-11-15T17:40:49,427 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0094df76678f4b3297d31f98af019a7e, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T17:40:49,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/d6505d0679dc4842b236077f700cfe33 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d6505d0679dc4842b236077f700cfe33 2024-11-15T17:40:49,432 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d6505d0679dc4842b236077f700cfe33, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T17:40:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/1b1b9913769b47fdb1ae2a36b2649d30 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1b1b9913769b47fdb1ae2a36b2649d30 2024-11-15T17:40:49,437 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1b1b9913769b47fdb1ae2a36b2649d30, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T17:40:49,438 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 482ms, sequenceid=205, compaction requested=false 2024-11-15T17:40:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T17:40:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T17:40:49,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T17:40:49,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0970 sec 2024-11-15T17:40:49,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1050 sec 2024-11-15T17:40:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T17:40:49,475 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:49,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T17:40:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T17:40:49,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:49,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:49,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T17:40:49,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T17:40:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:49,630 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T17:40:49,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/c910795d60304fdebc1f7c1264b814a8 is 50, key is test_row_0/A:col10/1731692449029/Put/seqid=0 2024-11-15T17:40:49,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742112_1288 (size=12151) 2024-11-15T17:40:49,640 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/c910795d60304fdebc1f7c1264b814a8 2024-11-15T17:40:49,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/cb2dba5339e14852a982789caebbbf80 is 50, key is test_row_0/B:col10/1731692449029/Put/seqid=0 2024-11-15T17:40:49,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742113_1289 (size=12151) 2024-11-15T17:40:49,652 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/cb2dba5339e14852a982789caebbbf80 2024-11-15T17:40:49,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/cf9d2dfe395f439aa41b2b6c1c9ff45d is 50, key is test_row_0/C:col10/1731692449029/Put/seqid=0 2024-11-15T17:40:49,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742114_1290 (size=12151) 2024-11-15T17:40:49,662 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/cf9d2dfe395f439aa41b2b6c1c9ff45d 2024-11-15T17:40:49,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/c910795d60304fdebc1f7c1264b814a8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c910795d60304fdebc1f7c1264b814a8 2024-11-15T17:40:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:49,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:49,670 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c910795d60304fdebc1f7c1264b814a8, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T17:40:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/cb2dba5339e14852a982789caebbbf80 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/cb2dba5339e14852a982789caebbbf80 2024-11-15T17:40:49,675 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/cb2dba5339e14852a982789caebbbf80, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T17:40:49,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/cf9d2dfe395f439aa41b2b6c1c9ff45d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/cf9d2dfe395f439aa41b2b6c1c9ff45d 2024-11-15T17:40:49,679 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/cf9d2dfe395f439aa41b2b6c1c9ff45d, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T17:40:49,680 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=20.13 KB/20610 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 50ms, sequenceid=232, compaction requested=true 2024-11-15T17:40:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T17:40:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T17:40:49,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T17:40:49,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 203 msec 2024-11-15T17:40:49,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 207 msec 2024-11-15T17:40:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:49,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:40:49,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/e9762a2bbd4f41ff909e9ec77ed046f5 is 50, key is test_row_0/A:col10/1731692449670/Put/seqid=0 2024-11-15T17:40:49,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742115_1291 (size=14541) 2024-11-15T17:40:49,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692509717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692509718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692509720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T17:40:49,795 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:49,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T17:40:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T17:40:49,797 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:49,798 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:49,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:49,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692509826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692509826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692509826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T17:40:49,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:49,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692510035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692510035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692510035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692510065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692510065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,068 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:50,068 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/e9762a2bbd4f41ff909e9ec77ed046f5 2024-11-15T17:40:50,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/d8bcaf0e56cc4459acc6e2996120af8a is 50, key is test_row_0/B:col10/1731692449670/Put/seqid=0 2024-11-15T17:40:50,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742116_1292 (size=12151) 2024-11-15T17:40:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T17:40:50,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692510345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692510345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692510345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T17:40:50,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/d8bcaf0e56cc4459acc6e2996120af8a 2024-11-15T17:40:50,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/57d4c57ebd004b6d85959d70697019e6 is 50, key is test_row_0/C:col10/1731692449670/Put/seqid=0 2024-11-15T17:40:50,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742117_1293 (size=12151) 2024-11-15T17:40:50,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692510855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692510855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692510855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:50,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:50,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/57d4c57ebd004b6d85959d70697019e6 2024-11-15T17:40:50,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/e9762a2bbd4f41ff909e9ec77ed046f5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/e9762a2bbd4f41ff909e9ec77ed046f5 2024-11-15T17:40:50,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/e9762a2bbd4f41ff909e9ec77ed046f5, entries=200, sequenceid=244, filesize=14.2 K 2024-11-15T17:40:50,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/d8bcaf0e56cc4459acc6e2996120af8a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d8bcaf0e56cc4459acc6e2996120af8a 2024-11-15T17:40:50,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d8bcaf0e56cc4459acc6e2996120af8a, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T17:40:50,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/57d4c57ebd004b6d85959d70697019e6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/57d4c57ebd004b6d85959d70697019e6 2024-11-15T17:40:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T17:40:50,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/57d4c57ebd004b6d85959d70697019e6, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T17:40:50,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1250ms, sequenceid=244, compaction requested=true 2024-11-15T17:40:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:50,937 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:50,937 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:50,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:50,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:50,938 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:50,938 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,938 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:50,939 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:50,939 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a3018de4a32d4ee48834dd2c59a7ed50, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d6505d0679dc4842b236077f700cfe33, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/cb2dba5339e14852a982789caebbbf80, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d8bcaf0e56cc4459acc6e2996120af8a] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=47.9 K 2024-11-15T17:40:50,939 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fde029ebe923486a86de43acff7cc52f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0094df76678f4b3297d31f98af019a7e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c910795d60304fdebc1f7c1264b814a8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/e9762a2bbd4f41ff909e9ec77ed046f5] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=50.2 K 2024-11-15T17:40:50,939 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting a3018de4a32d4ee48834dd2c59a7ed50, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731692447749 2024-11-15T17:40:50,939 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde029ebe923486a86de43acff7cc52f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731692447749 2024-11-15T17:40:50,939 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d6505d0679dc4842b236077f700cfe33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692447910 2024-11-15T17:40:50,939 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0094df76678f4b3297d31f98af019a7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692447910 2024-11-15T17:40:50,939 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2dba5339e14852a982789caebbbf80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731692449029 2024-11-15T17:40:50,940 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c910795d60304fdebc1f7c1264b814a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731692449029 2024-11-15T17:40:50,940 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d8bcaf0e56cc4459acc6e2996120af8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731692449670 2024-11-15T17:40:50,940 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9762a2bbd4f41ff909e9ec77ed046f5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731692449670 2024-11-15T17:40:50,948 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:50,948 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:50,948 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/3d75ddf8e0ec40eda710013d99c27d31 is 50, key is test_row_0/A:col10/1731692449670/Put/seqid=0 2024-11-15T17:40:50,948 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/fabd7d0135b6433f872be249c54bb4ce is 50, key is test_row_0/B:col10/1731692449670/Put/seqid=0 2024-11-15T17:40:50,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742118_1294 (size=12731) 2024-11-15T17:40:50,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742119_1295 (size=12731) 2024-11-15T17:40:51,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T17:40:51,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:51,019 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:40:51,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7f7ce79acc634ca3ab96604336d05094 is 50, key is test_row_0/A:col10/1731692449710/Put/seqid=0 2024-11-15T17:40:51,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742120_1296 (size=12301) 2024-11-15T17:40:51,365 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/fabd7d0135b6433f872be249c54bb4ce as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fabd7d0135b6433f872be249c54bb4ce 2024-11-15T17:40:51,369 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into fabd7d0135b6433f872be249c54bb4ce(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:51,369 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:51,369 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=12, startTime=1731692450937; duration=0sec 2024-11-15T17:40:51,369 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:51,369 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:51,369 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:51,370 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:51,370 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:51,370 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:51,371 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/c138e82d4fce4edea13db8ba213b2daf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1b1b9913769b47fdb1ae2a36b2649d30, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/cf9d2dfe395f439aa41b2b6c1c9ff45d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/57d4c57ebd004b6d85959d70697019e6] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=47.9 K 2024-11-15T17:40:51,371 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c138e82d4fce4edea13db8ba213b2daf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731692447749 2024-11-15T17:40:51,371 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1b9913769b47fdb1ae2a36b2649d30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692447910 2024-11-15T17:40:51,371 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9d2dfe395f439aa41b2b6c1c9ff45d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731692449029 2024-11-15T17:40:51,372 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d4c57ebd004b6d85959d70697019e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731692449670 2024-11-15T17:40:51,376 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/3d75ddf8e0ec40eda710013d99c27d31 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/3d75ddf8e0ec40eda710013d99c27d31 2024-11-15T17:40:51,381 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:51,381 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into 3d75ddf8e0ec40eda710013d99c27d31(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:51,381 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:51,381 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=12, startTime=1731692450937; duration=0sec 2024-11-15T17:40:51,381 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/33ad0763fe6842eeb3c161c3c7df12a0 is 50, key is test_row_0/C:col10/1731692449670/Put/seqid=0 2024-11-15T17:40:51,381 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:51,381 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:51,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742121_1297 (size=12731) 2024-11-15T17:40:51,428 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7f7ce79acc634ca3ab96604336d05094 2024-11-15T17:40:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/e0ca6d6cd06640d29431b74076a6bd47 is 50, key is test_row_0/B:col10/1731692449710/Put/seqid=0 2024-11-15T17:40:51,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742122_1298 (size=12301) 2024-11-15T17:40:51,439 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/e0ca6d6cd06640d29431b74076a6bd47 2024-11-15T17:40:51,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/0cc9f986249c4e4f8bc7fde8dc2593c2 is 50, key is test_row_0/C:col10/1731692449710/Put/seqid=0 2024-11-15T17:40:51,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742123_1299 (size=12301) 2024-11-15T17:40:51,451 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/0cc9f986249c4e4f8bc7fde8dc2593c2 2024-11-15T17:40:51,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7f7ce79acc634ca3ab96604336d05094 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7f7ce79acc634ca3ab96604336d05094 2024-11-15T17:40:51,463 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7f7ce79acc634ca3ab96604336d05094, entries=150, sequenceid=268, filesize=12.0 K 2024-11-15T17:40:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/e0ca6d6cd06640d29431b74076a6bd47 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/e0ca6d6cd06640d29431b74076a6bd47 2024-11-15T17:40:51,468 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/e0ca6d6cd06640d29431b74076a6bd47, entries=150, sequenceid=268, filesize=12.0 K 2024-11-15T17:40:51,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/0cc9f986249c4e4f8bc7fde8dc2593c2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0cc9f986249c4e4f8bc7fde8dc2593c2 2024-11-15T17:40:51,475 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0cc9f986249c4e4f8bc7fde8dc2593c2, entries=150, sequenceid=268, filesize=12.0 K 2024-11-15T17:40:51,475 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 456ms, sequenceid=268, compaction requested=false 2024-11-15T17:40:51,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:51,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:51,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T17:40:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T17:40:51,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T17:40:51,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6790 sec 2024-11-15T17:40:51,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6830 sec 2024-11-15T17:40:51,789 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/33ad0763fe6842eeb3c161c3c7df12a0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/33ad0763fe6842eeb3c161c3c7df12a0 2024-11-15T17:40:51,793 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into 33ad0763fe6842eeb3c161c3c7df12a0(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:51,793 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:51,793 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=12, startTime=1731692450937; duration=0sec 2024-11-15T17:40:51,793 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:51,793 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:51,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:51,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:40:51,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/8d6156d7d41444fd9e46580df774a4cf is 50, key is test_row_0/A:col10/1731692451873/Put/seqid=0 2024-11-15T17:40:51,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742124_1300 (size=12301) 2024-11-15T17:40:51,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692511903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:51,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692511905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:51,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692511905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T17:40:51,945 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:51,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T17:40:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T17:40:51,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:51,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:51,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:52,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692512016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692512016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692512016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T17:40:52,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692512075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,078 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692512095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,097 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692512225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692512226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692512226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:52,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T17:40:52,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/8d6156d7d41444fd9e46580df774a4cf 2024-11-15T17:40:52,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/187e74f735c442db890d75c15d1dc0a5 is 50, key is test_row_0/B:col10/1731692451873/Put/seqid=0 2024-11-15T17:40:52,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742125_1301 (size=12301) 2024-11-15T17:40:52,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:52,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:52,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692512535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692512535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692512535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:52,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:52,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:52,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T17:40:52,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/187e74f735c442db890d75c15d1dc0a5 2024-11-15T17:40:52,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/91410f8f9aaa48bbb8966d9fb0e19a5e is 50, key is test_row_0/C:col10/1731692451873/Put/seqid=0 2024-11-15T17:40:52,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742126_1302 (size=12301) 2024-11-15T17:40:52,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:52,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:52,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:52,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:53,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:53,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:53,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:53,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:53,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:53,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:53,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692513045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:53,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692513045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:53,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692513046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T17:40:53,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/91410f8f9aaa48bbb8966d9fb0e19a5e 2024-11-15T17:40:53,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/8d6156d7d41444fd9e46580df774a4cf as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8d6156d7d41444fd9e46580df774a4cf 2024-11-15T17:40:53,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8d6156d7d41444fd9e46580df774a4cf, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T17:40:53,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/187e74f735c442db890d75c15d1dc0a5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/187e74f735c442db890d75c15d1dc0a5 2024-11-15T17:40:53,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/187e74f735c442db890d75c15d1dc0a5, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T17:40:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/91410f8f9aaa48bbb8966d9fb0e19a5e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/91410f8f9aaa48bbb8966d9fb0e19a5e 2024-11-15T17:40:53,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/91410f8f9aaa48bbb8966d9fb0e19a5e, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T17:40:53,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1247ms, sequenceid=283, compaction requested=true 2024-11-15T17:40:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:53,121 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:53,121 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:53,122 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:53,122 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:53,122 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:53,122 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/3d75ddf8e0ec40eda710013d99c27d31, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7f7ce79acc634ca3ab96604336d05094, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8d6156d7d41444fd9e46580df774a4cf] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=36.5 K 2024-11-15T17:40:53,122 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:53,122 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:53,123 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:53,123 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fabd7d0135b6433f872be249c54bb4ce, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/e0ca6d6cd06640d29431b74076a6bd47, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/187e74f735c442db890d75c15d1dc0a5] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=36.5 K 2024-11-15T17:40:53,123 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d75ddf8e0ec40eda710013d99c27d31, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731692449670 2024-11-15T17:40:53,123 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fabd7d0135b6433f872be249c54bb4ce, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731692449670 2024-11-15T17:40:53,123 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7ce79acc634ca3ab96604336d05094, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731692449710 2024-11-15T17:40:53,123 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting e0ca6d6cd06640d29431b74076a6bd47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731692449710 2024-11-15T17:40:53,124 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 187e74f735c442db890d75c15d1dc0a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731692451870 2024-11-15T17:40:53,124 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d6156d7d41444fd9e46580df774a4cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731692451870 2024-11-15T17:40:53,132 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:53,132 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:53,133 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/8606febcdbbc4c38b8ed6d788c7fe523 is 50, key is test_row_0/A:col10/1731692451873/Put/seqid=0 2024-11-15T17:40:53,134 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/ce1dd3f13b784b5b952bd6df6b59ffc9 is 50, key is test_row_0/B:col10/1731692451873/Put/seqid=0 2024-11-15T17:40:53,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742127_1303 (size=12983) 2024-11-15T17:40:53,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742128_1304 (size=12983) 2024-11-15T17:40:53,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T17:40:53,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:53,170 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:40:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/49ffcd184b3d43388db3d5d7e513b5a7 is 50, key is test_row_0/A:col10/1731692451904/Put/seqid=0 2024-11-15T17:40:53,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742129_1305 (size=12301) 2024-11-15T17:40:53,555 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/8606febcdbbc4c38b8ed6d788c7fe523 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8606febcdbbc4c38b8ed6d788c7fe523 2024-11-15T17:40:53,564 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into 8606febcdbbc4c38b8ed6d788c7fe523(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:53,564 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:53,564 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=13, startTime=1731692453121; duration=0sec 2024-11-15T17:40:53,565 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:53,565 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:53,565 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:53,566 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:53,566 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:53,566 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:53,566 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/33ad0763fe6842eeb3c161c3c7df12a0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0cc9f986249c4e4f8bc7fde8dc2593c2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/91410f8f9aaa48bbb8966d9fb0e19a5e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=36.5 K 2024-11-15T17:40:53,569 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/ce1dd3f13b784b5b952bd6df6b59ffc9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/ce1dd3f13b784b5b952bd6df6b59ffc9 2024-11-15T17:40:53,571 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ad0763fe6842eeb3c161c3c7df12a0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731692449670 2024-11-15T17:40:53,572 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cc9f986249c4e4f8bc7fde8dc2593c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731692449710 2024-11-15T17:40:53,573 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91410f8f9aaa48bbb8966d9fb0e19a5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731692451870 2024-11-15T17:40:53,579 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into ce1dd3f13b784b5b952bd6df6b59ffc9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:53,579 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:53,579 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=13, startTime=1731692453121; duration=0sec 2024-11-15T17:40:53,579 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:53,579 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:53,581 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/49ffcd184b3d43388db3d5d7e513b5a7 2024-11-15T17:40:53,613 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:53,613 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/49412d85528d49bc9377555a932a2097 is 50, key is test_row_0/C:col10/1731692451873/Put/seqid=0 2024-11-15T17:40:53,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/c05e460d5e654b28b9ef9af2f3876ca6 is 50, key is test_row_0/B:col10/1731692451904/Put/seqid=0 2024-11-15T17:40:53,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742130_1306 (size=12983) 2024-11-15T17:40:53,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742131_1307 (size=12301) 2024-11-15T17:40:54,054 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/49412d85528d49bc9377555a932a2097 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/49412d85528d49bc9377555a932a2097 2024-11-15T17:40:54,057 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/c05e460d5e654b28b9ef9af2f3876ca6 2024-11-15T17:40:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:54,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:54,060 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into 49412d85528d49bc9377555a932a2097(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:54,060 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:54,060 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=13, startTime=1731692453121; duration=0sec 2024-11-15T17:40:54,061 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:54,061 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:54,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/7d3bb7d8f84942238608e40a729c10d1 is 50, key is test_row_0/C:col10/1731692451904/Put/seqid=0 2024-11-15T17:40:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742132_1308 (size=12301) 2024-11-15T17:40:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692514069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692514070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692514071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T17:40:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692514185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692514186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692514186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692514396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692514396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692514396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,472 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/7d3bb7d8f84942238608e40a729c10d1 2024-11-15T17:40:54,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/49ffcd184b3d43388db3d5d7e513b5a7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/49ffcd184b3d43388db3d5d7e513b5a7 2024-11-15T17:40:54,480 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/49ffcd184b3d43388db3d5d7e513b5a7, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T17:40:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/c05e460d5e654b28b9ef9af2f3876ca6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/c05e460d5e654b28b9ef9af2f3876ca6 2024-11-15T17:40:54,484 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/c05e460d5e654b28b9ef9af2f3876ca6, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T17:40:54,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/7d3bb7d8f84942238608e40a729c10d1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7d3bb7d8f84942238608e40a729c10d1 2024-11-15T17:40:54,488 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7d3bb7d8f84942238608e40a729c10d1, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T17:40:54,489 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1318ms, sequenceid=307, compaction requested=false 2024-11-15T17:40:54,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:54,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:54,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T17:40:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T17:40:54,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T17:40:54,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5420 sec 2024-11-15T17:40:54,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.5470 sec 2024-11-15T17:40:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:54,708 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:54,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:40:54,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/0cfaaaa4f00442ef9290a041685d40ef is 50, key is test_row_0/A:col10/1731692454708/Put/seqid=0 2024-11-15T17:40:54,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742133_1309 (size=14741) 2024-11-15T17:40:54,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692514738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692514739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692514740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692514846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692514846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:54,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692514856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692515056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692515056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692515065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/0cfaaaa4f00442ef9290a041685d40ef 2024-11-15T17:40:55,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/636966ed05054cd0bacc29043c63cc22 is 50, key is test_row_0/B:col10/1731692454708/Put/seqid=0 2024-11-15T17:40:55,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742134_1310 (size=12301) 2024-11-15T17:40:55,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/636966ed05054cd0bacc29043c63cc22 2024-11-15T17:40:55,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/38719834736946d7a7d2eb1585ac1e82 is 50, key is test_row_0/C:col10/1731692454708/Put/seqid=0 2024-11-15T17:40:55,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742135_1311 (size=12301) 2024-11-15T17:40:55,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692515366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692515366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692515375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/38719834736946d7a7d2eb1585ac1e82 2024-11-15T17:40:55,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/0cfaaaa4f00442ef9290a041685d40ef as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0cfaaaa4f00442ef9290a041685d40ef 2024-11-15T17:40:55,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0cfaaaa4f00442ef9290a041685d40ef, entries=200, sequenceid=325, filesize=14.4 K 2024-11-15T17:40:55,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/636966ed05054cd0bacc29043c63cc22 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/636966ed05054cd0bacc29043c63cc22 2024-11-15T17:40:55,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/636966ed05054cd0bacc29043c63cc22, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T17:40:55,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/38719834736946d7a7d2eb1585ac1e82 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/38719834736946d7a7d2eb1585ac1e82 2024-11-15T17:40:55,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/38719834736946d7a7d2eb1585ac1e82, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T17:40:55,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 875ms, sequenceid=325, compaction requested=true 2024-11-15T17:40:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:55,584 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:55,584 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:55,585 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:55,585 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:55,585 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:55,585 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:55,585 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:55,585 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:55,585 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8606febcdbbc4c38b8ed6d788c7fe523, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/49ffcd184b3d43388db3d5d7e513b5a7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0cfaaaa4f00442ef9290a041685d40ef] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=39.1 K 2024-11-15T17:40:55,585 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/ce1dd3f13b784b5b952bd6df6b59ffc9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/c05e460d5e654b28b9ef9af2f3876ca6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/636966ed05054cd0bacc29043c63cc22] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=36.7 K 2024-11-15T17:40:55,585 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8606febcdbbc4c38b8ed6d788c7fe523, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731692451870 2024-11-15T17:40:55,585 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1dd3f13b784b5b952bd6df6b59ffc9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731692451870 2024-11-15T17:40:55,585 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49ffcd184b3d43388db3d5d7e513b5a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731692451897 2024-11-15T17:40:55,586 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c05e460d5e654b28b9ef9af2f3876ca6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731692451897 2024-11-15T17:40:55,586 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cfaaaa4f00442ef9290a041685d40ef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731692454068 2024-11-15T17:40:55,586 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 636966ed05054cd0bacc29043c63cc22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731692454068 2024-11-15T17:40:55,596 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:55,597 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f85e7d554c624e5ba18d5178d67006a6 is 50, key is test_row_0/A:col10/1731692454708/Put/seqid=0 2024-11-15T17:40:55,599 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:55,600 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/7d9535c238f4424d86491d22ab8f9a39 is 50, key is test_row_0/B:col10/1731692454708/Put/seqid=0 2024-11-15T17:40:55,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742136_1312 (size=13085) 2024-11-15T17:40:55,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742137_1313 (size=13085) 2024-11-15T17:40:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:55,878 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:55,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:40:55,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/cdd34c8e5034457da23682b7b8152336 is 50, key is test_row_0/A:col10/1731692455878/Put/seqid=0 2024-11-15T17:40:55,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742138_1314 (size=17181) 2024-11-15T17:40:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692515896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692515898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:55,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692515899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692516005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692516005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692516006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,017 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f85e7d554c624e5ba18d5178d67006a6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f85e7d554c624e5ba18d5178d67006a6 2024-11-15T17:40:56,022 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into f85e7d554c624e5ba18d5178d67006a6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:56,023 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:56,023 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=13, startTime=1731692455584; duration=0sec 2024-11-15T17:40:56,023 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:56,023 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:56,023 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:40:56,025 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:40:56,025 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:56,025 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,025 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/49412d85528d49bc9377555a932a2097, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7d3bb7d8f84942238608e40a729c10d1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/38719834736946d7a7d2eb1585ac1e82] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=36.7 K 2024-11-15T17:40:56,026 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49412d85528d49bc9377555a932a2097, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731692451870 2024-11-15T17:40:56,026 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3bb7d8f84942238608e40a729c10d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731692451897 2024-11-15T17:40:56,027 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38719834736946d7a7d2eb1585ac1e82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731692454068 2024-11-15T17:40:56,034 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:56,035 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/32d52134896e4ada96a65be6354c5567 is 50, key is test_row_0/C:col10/1731692454708/Put/seqid=0 2024-11-15T17:40:56,055 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/7d9535c238f4424d86491d22ab8f9a39 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/7d9535c238f4424d86491d22ab8f9a39 2024-11-15T17:40:56,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742139_1315 (size=13085) 2024-11-15T17:40:56,062 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into 7d9535c238f4424d86491d22ab8f9a39(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:56,062 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:56,062 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=13, startTime=1731692455584; duration=0sec 2024-11-15T17:40:56,062 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:56,062 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:56,067 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/32d52134896e4ada96a65be6354c5567 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/32d52134896e4ada96a65be6354c5567 2024-11-15T17:40:56,072 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into 32d52134896e4ada96a65be6354c5567(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:56,072 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:56,072 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=13, startTime=1731692455584; duration=0sec 2024-11-15T17:40:56,072 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:56,072 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T17:40:56,106 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:56,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-15T17:40:56,108 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T17:40:56,110 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:56,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:56,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48532 deadline: 1731692516116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,118 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48572 deadline: 1731692516125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,128 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 96ad00ac4dbe2b71b39f7138f6d0e503 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T17:40:56,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692516216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692516216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692516217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T17:40:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/cdd34c8e5034457da23682b7b8152336 2024-11-15T17:40:56,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/100490393ef34fb09c9837f2c9a8fcae is 50, key is test_row_0/B:col10/1731692455878/Put/seqid=0 2024-11-15T17:40:56,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742140_1316 (size=12301) 2024-11-15T17:40:56,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T17:40:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T17:40:56,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692516525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692516525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692516525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:56,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T17:40:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/100490393ef34fb09c9837f2c9a8fcae 2024-11-15T17:40:56,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/151f23da4a664bc8824297b352fa0756 is 50, key is test_row_0/C:col10/1731692455878/Put/seqid=0 2024-11-15T17:40:56,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742141_1317 (size=12301) 2024-11-15T17:40:56,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/151f23da4a664bc8824297b352fa0756 2024-11-15T17:40:56,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T17:40:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/cdd34c8e5034457da23682b7b8152336 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cdd34c8e5034457da23682b7b8152336 2024-11-15T17:40:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:56,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cdd34c8e5034457da23682b7b8152336, entries=250, sequenceid=348, filesize=16.8 K 2024-11-15T17:40:56,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/100490393ef34fb09c9837f2c9a8fcae as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/100490393ef34fb09c9837f2c9a8fcae 2024-11-15T17:40:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T17:40:56,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/100490393ef34fb09c9837f2c9a8fcae, entries=150, sequenceid=348, filesize=12.0 K 2024-11-15T17:40:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/151f23da4a664bc8824297b352fa0756 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/151f23da4a664bc8824297b352fa0756 2024-11-15T17:40:56,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/151f23da4a664bc8824297b352fa0756, entries=150, sequenceid=348, filesize=12.0 K 2024-11-15T17:40:56,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 872ms, sequenceid=348, compaction requested=false 2024-11-15T17:40:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:56,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T17:40:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:56,877 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:40:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/145f7f90f3634964a60d5a8d43d41c86 is 50, key is test_row_0/A:col10/1731692455896/Put/seqid=0 2024-11-15T17:40:56,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742142_1318 (size=12301) 2024-11-15T17:40:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:57,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:57,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692517063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692517065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692517065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692517175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692517175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692517175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T17:40:57,284 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/145f7f90f3634964a60d5a8d43d41c86 2024-11-15T17:40:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/9d55c08e3b104e5f91da3d64c3bf738f is 50, key is test_row_0/B:col10/1731692455896/Put/seqid=0 2024-11-15T17:40:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742143_1319 (size=12301) 2024-11-15T17:40:57,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692517385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692517385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692517385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,694 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/9d55c08e3b104e5f91da3d64c3bf738f 2024-11-15T17:40:57,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692517696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692517696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692517699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/6787474603e84aeab435319d087b2348 is 50, key is test_row_0/C:col10/1731692455896/Put/seqid=0 2024-11-15T17:40:57,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742144_1320 (size=12301) 2024-11-15T17:40:58,118 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/6787474603e84aeab435319d087b2348 2024-11-15T17:40:58,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/145f7f90f3634964a60d5a8d43d41c86 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/145f7f90f3634964a60d5a8d43d41c86 2024-11-15T17:40:58,129 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/145f7f90f3634964a60d5a8d43d41c86, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T17:40:58,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/9d55c08e3b104e5f91da3d64c3bf738f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/9d55c08e3b104e5f91da3d64c3bf738f 2024-11-15T17:40:58,134 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/9d55c08e3b104e5f91da3d64c3bf738f, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T17:40:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/6787474603e84aeab435319d087b2348 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/6787474603e84aeab435319d087b2348 2024-11-15T17:40:58,139 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/6787474603e84aeab435319d087b2348, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T17:40:58,140 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1263ms, sequenceid=364, compaction requested=true 2024-11-15T17:40:58,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:58,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-15T17:40:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-15T17:40:58,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T17:40:58,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0320 sec 2024-11-15T17:40:58,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.0390 sec 2024-11-15T17:40:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:58,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:40:58,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f71a5c0f1581489899af8fa9e7a0656a is 50, key is test_row_0/A:col10/1731692458209/Put/seqid=0 2024-11-15T17:40:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742145_1321 (size=17181) 2024-11-15T17:40:58,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f71a5c0f1581489899af8fa9e7a0656a 2024-11-15T17:40:58,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/060cb40591cc4e23b21f5adaa6ac7638 is 50, key is test_row_0/B:col10/1731692458209/Put/seqid=0 2024-11-15T17:40:58,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742146_1322 (size=12301) 2024-11-15T17:40:58,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/060cb40591cc4e23b21f5adaa6ac7638 2024-11-15T17:40:58,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/0e689088f3cb4a91bc532db71836ac5a is 50, key is test_row_0/C:col10/1731692458209/Put/seqid=0 2024-11-15T17:40:58,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742147_1323 (size=12301) 2024-11-15T17:40:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T17:40:58,256 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:58,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-15T17:40:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T17:40:58,258 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:58,259 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:58,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:58,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692518261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692518261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692518263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T17:40:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692518376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692518376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692518376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T17:40:58,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:58,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:58,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:58,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T17:40:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:58,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T17:40:58,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692518585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692518586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692518586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:58,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/0e689088f3cb4a91bc532db71836ac5a 2024-11-15T17:40:58,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f71a5c0f1581489899af8fa9e7a0656a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f71a5c0f1581489899af8fa9e7a0656a 2024-11-15T17:40:58,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f71a5c0f1581489899af8fa9e7a0656a, entries=250, sequenceid=391, filesize=16.8 K 2024-11-15T17:40:58,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/060cb40591cc4e23b21f5adaa6ac7638 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/060cb40591cc4e23b21f5adaa6ac7638 2024-11-15T17:40:58,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/060cb40591cc4e23b21f5adaa6ac7638, entries=150, sequenceid=391, filesize=12.0 K 2024-11-15T17:40:58,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/0e689088f3cb4a91bc532db71836ac5a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0e689088f3cb4a91bc532db71836ac5a 2024-11-15T17:40:58,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0e689088f3cb4a91bc532db71836ac5a, entries=150, sequenceid=391, filesize=12.0 K 2024-11-15T17:40:58,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 471ms, sequenceid=391, compaction requested=true 2024-11-15T17:40:58,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:58,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:40:58,681 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:58,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:58,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:40:58,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:58,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:40:58,681 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:40:58,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:58,688 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:58,688 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:40:58,689 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,689 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/7d9535c238f4424d86491d22ab8f9a39, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/100490393ef34fb09c9837f2c9a8fcae, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/9d55c08e3b104e5f91da3d64c3bf738f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/060cb40591cc4e23b21f5adaa6ac7638] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=48.8 K 2024-11-15T17:40:58,689 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:40:58,689 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d9535c238f4424d86491d22ab8f9a39, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731692454068 2024-11-15T17:40:58,689 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:40:58,690 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,690 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f85e7d554c624e5ba18d5178d67006a6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cdd34c8e5034457da23682b7b8152336, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/145f7f90f3634964a60d5a8d43d41c86, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f71a5c0f1581489899af8fa9e7a0656a] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=58.3 K 2024-11-15T17:40:58,690 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 100490393ef34fb09c9837f2c9a8fcae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731692454738 2024-11-15T17:40:58,690 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85e7d554c624e5ba18d5178d67006a6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731692454068 2024-11-15T17:40:58,690 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdd34c8e5034457da23682b7b8152336, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731692454737 2024-11-15T17:40:58,691 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d55c08e3b104e5f91da3d64c3bf738f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731692455894 2024-11-15T17:40:58,691 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145f7f90f3634964a60d5a8d43d41c86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731692455894 2024-11-15T17:40:58,691 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 060cb40591cc4e23b21f5adaa6ac7638, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731692457062 2024-11-15T17:40:58,692 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f71a5c0f1581489899af8fa9e7a0656a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731692457062 2024-11-15T17:40:58,717 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:58,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T17:40:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,718 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T17:40:58,718 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/fa27b8cdd3ff4f89b73b7deea38ee068 is 50, key is test_row_0/A:col10/1731692458209/Put/seqid=0 2024-11-15T17:40:58,725 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:58,726 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/67f20883a707428f82dc64ba7a8a2fea is 50, key is test_row_0/B:col10/1731692458209/Put/seqid=0 2024-11-15T17:40:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7c89aa6ecefb4da98d97d6d73c3f9f60 is 50, key is test_row_0/A:col10/1731692458229/Put/seqid=0 2024-11-15T17:40:58,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742149_1325 (size=13221) 2024-11-15T17:40:58,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742148_1324 (size=13221) 2024-11-15T17:40:58,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742150_1326 (size=12301) 2024-11-15T17:40:58,845 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7c89aa6ecefb4da98d97d6d73c3f9f60 2024-11-15T17:40:58,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/845e46de090845eebf42f8dac22ac780 is 50, key is test_row_0/B:col10/1731692458229/Put/seqid=0 2024-11-15T17:40:58,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742151_1327 (size=12301) 2024-11-15T17:40:58,885 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/845e46de090845eebf42f8dac22ac780 2024-11-15T17:40:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T17:40:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/244fceef8f8e47528a072d75e5c81a07 is 50, key is test_row_0/C:col10/1731692458229/Put/seqid=0 2024-11-15T17:40:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:58,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742152_1328 (size=12301) 2024-11-15T17:40:58,914 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/244fceef8f8e47528a072d75e5c81a07 2024-11-15T17:40:58,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/7c89aa6ecefb4da98d97d6d73c3f9f60 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7c89aa6ecefb4da98d97d6d73c3f9f60 2024-11-15T17:40:58,927 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7c89aa6ecefb4da98d97d6d73c3f9f60, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:40:58,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/845e46de090845eebf42f8dac22ac780 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/845e46de090845eebf42f8dac22ac780 2024-11-15T17:40:58,937 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/845e46de090845eebf42f8dac22ac780, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:40:58,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/244fceef8f8e47528a072d75e5c81a07 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/244fceef8f8e47528a072d75e5c81a07 2024-11-15T17:40:58,947 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/244fceef8f8e47528a072d75e5c81a07, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:40:58,949 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=107.34 KB/109920 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 231ms, sequenceid=400, compaction requested=true 2024-11-15T17:40:58,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:58,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:58,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-15T17:40:58,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:40:58,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:40:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-15T17:40:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:40:58,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T17:40:58,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 693 msec 2024-11-15T17:40:58,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 699 msec 2024-11-15T17:40:58,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ab620841face4f51927aa620cb5652a2 is 50, key is test_row_0/A:col10/1731692458945/Put/seqid=0 2024-11-15T17:40:58,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692518988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:58,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742153_1329 (size=19621) 2024-11-15T17:40:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692518998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692518998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692519105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692519105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692519116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,206 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/67f20883a707428f82dc64ba7a8a2fea as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/67f20883a707428f82dc64ba7a8a2fea 2024-11-15T17:40:59,210 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into 67f20883a707428f82dc64ba7a8a2fea(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:59,210 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:59,210 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=12, startTime=1731692458681; duration=0sec 2024-11-15T17:40:59,210 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:40:59,210 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:40:59,210 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T17:40:59,211 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T17:40:59,211 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/C is initiating minor compaction (all files) 2024-11-15T17:40:59,211 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/C in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:59,211 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/32d52134896e4ada96a65be6354c5567, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/151f23da4a664bc8824297b352fa0756, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/6787474603e84aeab435319d087b2348, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0e689088f3cb4a91bc532db71836ac5a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/244fceef8f8e47528a072d75e5c81a07] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=60.8 K 2024-11-15T17:40:59,212 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 32d52134896e4ada96a65be6354c5567, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731692454068 2024-11-15T17:40:59,212 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 151f23da4a664bc8824297b352fa0756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731692454738 2024-11-15T17:40:59,213 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6787474603e84aeab435319d087b2348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731692455894 2024-11-15T17:40:59,213 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e689088f3cb4a91bc532db71836ac5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731692457062 2024-11-15T17:40:59,213 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 244fceef8f8e47528a072d75e5c81a07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731692458216 2024-11-15T17:40:59,217 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/fa27b8cdd3ff4f89b73b7deea38ee068 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fa27b8cdd3ff4f89b73b7deea38ee068 2024-11-15T17:40:59,222 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:40:59,223 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/4c17c933d4334538b37d98b118317157 is 50, key is test_row_0/C:col10/1731692458229/Put/seqid=0 2024-11-15T17:40:59,223 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into fa27b8cdd3ff4f89b73b7deea38ee068(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:59,223 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:59,223 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=12, startTime=1731692458681; duration=0sec 2024-11-15T17:40:59,224 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:59,224 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:40:59,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742154_1330 (size=13255) 2024-11-15T17:40:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692519315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692519315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692519325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T17:40:59,395 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:40:59,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:40:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-15T17:40:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T17:40:59,398 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:40:59,398 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:40:59,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:40:59,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ab620841face4f51927aa620cb5652a2 2024-11-15T17:40:59,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/59a778aa0868452ba1cf32b02a5dec84 is 50, key is test_row_0/B:col10/1731692458945/Put/seqid=0 2024-11-15T17:40:59,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742155_1331 (size=12301) 2024-11-15T17:40:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T17:40:59,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T17:40:59,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:59,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:59,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:59,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692519625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692519625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,630 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/4c17c933d4334538b37d98b118317157 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/4c17c933d4334538b37d98b118317157 2024-11-15T17:40:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:40:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692519635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:40:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:40:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:40:59,640 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/C of 96ad00ac4dbe2b71b39f7138f6d0e503 into 4c17c933d4334538b37d98b118317157(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:40:59,640 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:40:59,640 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/C, priority=11, startTime=1731692458681; duration=0sec 2024-11-15T17:40:59,640 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:40:59,640 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:40:59,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T17:40:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:59,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T17:40:59,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/59a778aa0868452ba1cf32b02a5dec84 2024-11-15T17:40:59,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/e4617c7db95a4a7c8baae0e2549a6865 is 50, key is test_row_0/C:col10/1731692458945/Put/seqid=0 2024-11-15T17:40:59,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742156_1332 (size=12301) 2024-11-15T17:40:59,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T17:40:59,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:59,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:40:59,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:40:59,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:00,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T17:41:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:41:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:00,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:00,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T17:41:00,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48610 deadline: 1731692520135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:00,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48622 deadline: 1731692520135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48602 deadline: 1731692520146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:00,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T17:41:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:41:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:00,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:00,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/e4617c7db95a4a7c8baae0e2549a6865 2024-11-15T17:41:00,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ab620841face4f51927aa620cb5652a2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ab620841face4f51927aa620cb5652a2 2024-11-15T17:41:00,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ab620841face4f51927aa620cb5652a2, entries=300, sequenceid=420, filesize=19.2 K 2024-11-15T17:41:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/59a778aa0868452ba1cf32b02a5dec84 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/59a778aa0868452ba1cf32b02a5dec84 2024-11-15T17:41:00,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/59a778aa0868452ba1cf32b02a5dec84, entries=150, sequenceid=420, filesize=12.0 K 2024-11-15T17:41:00,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/e4617c7db95a4a7c8baae0e2549a6865 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/e4617c7db95a4a7c8baae0e2549a6865 2024-11-15T17:41:00,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/e4617c7db95a4a7c8baae0e2549a6865, entries=150, sequenceid=420, filesize=12.0 K 2024-11-15T17:41:00,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1284ms, sequenceid=420, compaction requested=true 2024-11-15T17:41:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:41:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:00,234 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:00,234 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96ad00ac4dbe2b71b39f7138f6d0e503:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:00,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:00,235 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45143 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:00,235 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:00,235 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/A is initiating minor compaction (all files) 2024-11-15T17:41:00,235 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 96ad00ac4dbe2b71b39f7138f6d0e503/B is initiating minor compaction (all files) 2024-11-15T17:41:00,235 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/A in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:00,235 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96ad00ac4dbe2b71b39f7138f6d0e503/B in TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:00,235 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fa27b8cdd3ff4f89b73b7deea38ee068, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7c89aa6ecefb4da98d97d6d73c3f9f60, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ab620841face4f51927aa620cb5652a2] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=44.1 K 2024-11-15T17:41:00,235 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/67f20883a707428f82dc64ba7a8a2fea, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/845e46de090845eebf42f8dac22ac780, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/59a778aa0868452ba1cf32b02a5dec84] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp, totalSize=36.9 K 2024-11-15T17:41:00,236 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa27b8cdd3ff4f89b73b7deea38ee068, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731692457062 2024-11-15T17:41:00,236 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f20883a707428f82dc64ba7a8a2fea, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731692457062 2024-11-15T17:41:00,236 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c89aa6ecefb4da98d97d6d73c3f9f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731692458216 2024-11-15T17:41:00,236 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 845e46de090845eebf42f8dac22ac780, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731692458216 2024-11-15T17:41:00,236 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab620841face4f51927aa620cb5652a2, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731692458906 2024-11-15T17:41:00,237 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a778aa0868452ba1cf32b02a5dec84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731692458934 2024-11-15T17:41:00,244 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#A#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:00,245 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96ad00ac4dbe2b71b39f7138f6d0e503#B#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:00,245 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ce4c6456046c4c21a9861905007b312c is 50, key is test_row_0/A:col10/1731692458945/Put/seqid=0 2024-11-15T17:41:00,245 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/74cfb151802f4405a73122fc5f587bdc is 50, key is test_row_0/B:col10/1731692458945/Put/seqid=0 2024-11-15T17:41:00,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742157_1333 (size=13323) 2024-11-15T17:41:00,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742158_1334 (size=13323) 2024-11-15T17:41:00,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T17:41:00,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:00,315 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T17:41:00,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f4c6e52325d8445e97e2becbee784564 is 50, key is test_row_0/A:col10/1731692458986/Put/seqid=0 2024-11-15T17:41:00,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742159_1335 (size=12301) 2024-11-15T17:41:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T17:41:00,652 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/ce4c6456046c4c21a9861905007b312c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ce4c6456046c4c21a9861905007b312c 2024-11-15T17:41:00,652 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/74cfb151802f4405a73122fc5f587bdc as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/74cfb151802f4405a73122fc5f587bdc 2024-11-15T17:41:00,658 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/A of 96ad00ac4dbe2b71b39f7138f6d0e503 into ce4c6456046c4c21a9861905007b312c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:00,658 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:41:00,658 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/A, priority=13, startTime=1731692460234; duration=0sec 2024-11-15T17:41:00,658 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:00,658 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:A 2024-11-15T17:41:00,658 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T17:41:00,659 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96ad00ac4dbe2b71b39f7138f6d0e503/B of 96ad00ac4dbe2b71b39f7138f6d0e503 into 74cfb151802f4405a73122fc5f587bdc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:00,659 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:41:00,659 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., storeName=96ad00ac4dbe2b71b39f7138f6d0e503/B, priority=13, startTime=1731692460234; duration=0sec 2024-11-15T17:41:00,659 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:00,659 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:B 2024-11-15T17:41:00,659 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:41:00,659 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:41:00,659 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. because compaction request was cancelled 2024-11-15T17:41:00,659 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96ad00ac4dbe2b71b39f7138f6d0e503:C 2024-11-15T17:41:00,723 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f4c6e52325d8445e97e2becbee784564 2024-11-15T17:41:00,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/a71c6e920f7d4341a925b4972dcff53a is 50, key is test_row_0/B:col10/1731692458986/Put/seqid=0 2024-11-15T17:41:00,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742160_1336 (size=12301) 2024-11-15T17:41:00,739 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-15T17:41:00,739 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:00,740 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,740 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:00,741 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-15T17:41:00,741 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:00,741 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,741 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-15T17:41:00,741 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:00,741 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,741 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:00,741 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:00,742 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-15T17:41:00,742 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:00,742 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,742 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:00,743 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-15T17:41:00,743 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:00,743 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,743 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:00,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:01,134 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/a71c6e920f7d4341a925b4972dcff53a 2024-11-15T17:41:01,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/a1947ba70ef241cda167dea6163d174f is 50, key is test_row_0/C:col10/1731692458986/Put/seqid=0 2024-11-15T17:41:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:41:01,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. as already flushing 2024-11-15T17:41:01,146 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-15T17:41:01,146 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:01,147 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:01,147 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:01,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:01,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742161_1337 (size=12301) 2024-11-15T17:41:01,157 INFO [Thread-1127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1127. 2024-11-15T17:41:01,157 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-15T17:41:01,157 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:01,157 DEBUG [Thread-1127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:01,157 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:01,157 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:01,157 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:01,157 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:01,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:01,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T17:41:01,550 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/a1947ba70ef241cda167dea6163d174f 2024-11-15T17:41:01,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/f4c6e52325d8445e97e2becbee784564 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f4c6e52325d8445e97e2becbee784564 2024-11-15T17:41:01,564 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f4c6e52325d8445e97e2becbee784564, entries=150, sequenceid=440, filesize=12.0 K 2024-11-15T17:41:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/a71c6e920f7d4341a925b4972dcff53a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a71c6e920f7d4341a925b4972dcff53a 2024-11-15T17:41:01,569 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a71c6e920f7d4341a925b4972dcff53a, entries=150, sequenceid=440, filesize=12.0 K 2024-11-15T17:41:01,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/a1947ba70ef241cda167dea6163d174f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a1947ba70ef241cda167dea6163d174f 2024-11-15T17:41:01,573 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a1947ba70ef241cda167dea6163d174f, entries=150, sequenceid=440, filesize=12.0 K 2024-11-15T17:41:01,575 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=20.13 KB/20610 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1259ms, sequenceid=440, compaction requested=true 2024-11-15T17:41:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 96ad00ac4dbe2b71b39f7138f6d0e503: 2024-11-15T17:41:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-15T17:41:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-15T17:41:01,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T17:41:01,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-15T17:41:01,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.1830 sec 2024-11-15T17:41:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T17:41:03,557 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:06,150 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-15T17:41:06,150 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-15T17:41:06,150 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:06,150 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:06,150 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:06,150 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:06,151 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:06,151 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:06,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:06,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3912 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3758 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3758 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3909 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3739 2024-11-15T17:41:06,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:41:06,153 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:41:06,153 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:41:06,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:06,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:06,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:06,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:41:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T17:41:06,159 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692466159"}]},"ts":"1731692466159"} 2024-11-15T17:41:06,160 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:41:06,160 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:41:06,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:41:06,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, UNASSIGN}] 2024-11-15T17:41:06,164 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, UNASSIGN 2024-11-15T17:41:06,164 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=96ad00ac4dbe2b71b39f7138f6d0e503, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:06,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, UNASSIGN because future has completed 2024-11-15T17:41:06,166 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:41:06,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:41:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T17:41:06,320 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:41:06,320 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:41:06,320 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 96ad00ac4dbe2b71b39f7138f6d0e503, disabling compactions & flushes 2024-11-15T17:41:06,320 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:06,320 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:06,320 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. after waiting 0 ms 2024-11-15T17:41:06,320 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:06,320 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 96ad00ac4dbe2b71b39f7138f6d0e503 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T17:41:06,329 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/cd3217d548814be58ffbbfeb120ae2e6 is 50, key is test_row_0/A:col10/1731692466147/Put/seqid=0 2024-11-15T17:41:06,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742162_1338 (size=12301) 2024-11-15T17:41:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T17:41:06,735 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/cd3217d548814be58ffbbfeb120ae2e6 2024-11-15T17:41:06,741 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/de667f50e06e4b419049f3882b6c60a5 is 50, key is test_row_0/B:col10/1731692466147/Put/seqid=0 2024-11-15T17:41:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742163_1339 (size=12301) 2024-11-15T17:41:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T17:41:07,146 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/de667f50e06e4b419049f3882b6c60a5 2024-11-15T17:41:07,160 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/a3df1141866c4ea6981011224aff3c14 is 50, key is test_row_0/C:col10/1731692466147/Put/seqid=0 2024-11-15T17:41:07,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742164_1340 (size=12301) 2024-11-15T17:41:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T17:41:07,565 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/a3df1141866c4ea6981011224aff3c14 2024-11-15T17:41:07,569 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/A/cd3217d548814be58ffbbfeb120ae2e6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cd3217d548814be58ffbbfeb120ae2e6 2024-11-15T17:41:07,573 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cd3217d548814be58ffbbfeb120ae2e6, entries=150, sequenceid=450, filesize=12.0 K 2024-11-15T17:41:07,573 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/B/de667f50e06e4b419049f3882b6c60a5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/de667f50e06e4b419049f3882b6c60a5 2024-11-15T17:41:07,577 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/de667f50e06e4b419049f3882b6c60a5, entries=150, sequenceid=450, filesize=12.0 K 2024-11-15T17:41:07,577 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/.tmp/C/a3df1141866c4ea6981011224aff3c14 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a3df1141866c4ea6981011224aff3c14 2024-11-15T17:41:07,580 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a3df1141866c4ea6981011224aff3c14, entries=150, sequenceid=450, filesize=12.0 K 2024-11-15T17:41:07,581 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1261ms, sequenceid=450, compaction requested=true 2024-11-15T17:41:07,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ae0d7712e7fa4da99c0b0057ed682ef4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7782eab5d71a4ae783486b8dc88f64f0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/565e9f33cc974591881e78cac3ac30de, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/408a3174674a4eb1bda0fc2cfda20733, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/50ee1d55c1c84acb918fe7ab22bd6536, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9958d727425f4ad7a392f8379e7a4151, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c7427b94dfd8464b995a84eca181de85, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/caaf19b2cbe846d0902325d01922a605, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/58c780906b1846d783f5cecdbd842d8f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/024be5f1469c463fa6fdb4c250398490, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9c9ae0ccca404f7b819c730fda651bfa, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fde029ebe923486a86de43acff7cc52f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/d7341c389fb345efa11c0756f06a9e91, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0094df76678f4b3297d31f98af019a7e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c910795d60304fdebc1f7c1264b814a8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/e9762a2bbd4f41ff909e9ec77ed046f5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/3d75ddf8e0ec40eda710013d99c27d31, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7f7ce79acc634ca3ab96604336d05094, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8606febcdbbc4c38b8ed6d788c7fe523, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8d6156d7d41444fd9e46580df774a4cf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/49ffcd184b3d43388db3d5d7e513b5a7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0cfaaaa4f00442ef9290a041685d40ef, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f85e7d554c624e5ba18d5178d67006a6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cdd34c8e5034457da23682b7b8152336, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/145f7f90f3634964a60d5a8d43d41c86, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f71a5c0f1581489899af8fa9e7a0656a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fa27b8cdd3ff4f89b73b7deea38ee068, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7c89aa6ecefb4da98d97d6d73c3f9f60, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ab620841face4f51927aa620cb5652a2] to archive 2024-11-15T17:41:07,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:41:07,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ae0d7712e7fa4da99c0b0057ed682ef4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ae0d7712e7fa4da99c0b0057ed682ef4 2024-11-15T17:41:07,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7782eab5d71a4ae783486b8dc88f64f0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7782eab5d71a4ae783486b8dc88f64f0 2024-11-15T17:41:07,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fdcb3fd0f61d41b0a6fc9f8c0d8ada2e 2024-11-15T17:41:07,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/565e9f33cc974591881e78cac3ac30de to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/565e9f33cc974591881e78cac3ac30de 2024-11-15T17:41:07,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/408a3174674a4eb1bda0fc2cfda20733 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/408a3174674a4eb1bda0fc2cfda20733 2024-11-15T17:41:07,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/50ee1d55c1c84acb918fe7ab22bd6536 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/50ee1d55c1c84acb918fe7ab22bd6536 2024-11-15T17:41:07,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9958d727425f4ad7a392f8379e7a4151 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9958d727425f4ad7a392f8379e7a4151 2024-11-15T17:41:07,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c7427b94dfd8464b995a84eca181de85 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c7427b94dfd8464b995a84eca181de85 2024-11-15T17:41:07,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/caaf19b2cbe846d0902325d01922a605 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/caaf19b2cbe846d0902325d01922a605 2024-11-15T17:41:07,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/58c780906b1846d783f5cecdbd842d8f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/58c780906b1846d783f5cecdbd842d8f 2024-11-15T17:41:07,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/024be5f1469c463fa6fdb4c250398490 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/024be5f1469c463fa6fdb4c250398490 2024-11-15T17:41:07,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9c9ae0ccca404f7b819c730fda651bfa to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/9c9ae0ccca404f7b819c730fda651bfa 2024-11-15T17:41:07,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fde029ebe923486a86de43acff7cc52f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fde029ebe923486a86de43acff7cc52f 2024-11-15T17:41:07,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/d7341c389fb345efa11c0756f06a9e91 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/d7341c389fb345efa11c0756f06a9e91 2024-11-15T17:41:07,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0094df76678f4b3297d31f98af019a7e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0094df76678f4b3297d31f98af019a7e 2024-11-15T17:41:07,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c910795d60304fdebc1f7c1264b814a8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/c910795d60304fdebc1f7c1264b814a8 2024-11-15T17:41:07,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/e9762a2bbd4f41ff909e9ec77ed046f5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/e9762a2bbd4f41ff909e9ec77ed046f5 2024-11-15T17:41:07,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/3d75ddf8e0ec40eda710013d99c27d31 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/3d75ddf8e0ec40eda710013d99c27d31 2024-11-15T17:41:07,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7f7ce79acc634ca3ab96604336d05094 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7f7ce79acc634ca3ab96604336d05094 2024-11-15T17:41:07,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8606febcdbbc4c38b8ed6d788c7fe523 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8606febcdbbc4c38b8ed6d788c7fe523 2024-11-15T17:41:07,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8d6156d7d41444fd9e46580df774a4cf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/8d6156d7d41444fd9e46580df774a4cf 2024-11-15T17:41:07,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/49ffcd184b3d43388db3d5d7e513b5a7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/49ffcd184b3d43388db3d5d7e513b5a7 2024-11-15T17:41:07,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0cfaaaa4f00442ef9290a041685d40ef to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/0cfaaaa4f00442ef9290a041685d40ef 2024-11-15T17:41:07,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f85e7d554c624e5ba18d5178d67006a6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f85e7d554c624e5ba18d5178d67006a6 2024-11-15T17:41:07,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cdd34c8e5034457da23682b7b8152336 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cdd34c8e5034457da23682b7b8152336 2024-11-15T17:41:07,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/145f7f90f3634964a60d5a8d43d41c86 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/145f7f90f3634964a60d5a8d43d41c86 2024-11-15T17:41:07,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f71a5c0f1581489899af8fa9e7a0656a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f71a5c0f1581489899af8fa9e7a0656a 2024-11-15T17:41:07,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fa27b8cdd3ff4f89b73b7deea38ee068 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/fa27b8cdd3ff4f89b73b7deea38ee068 2024-11-15T17:41:07,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7c89aa6ecefb4da98d97d6d73c3f9f60 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/7c89aa6ecefb4da98d97d6d73c3f9f60 2024-11-15T17:41:07,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ab620841face4f51927aa620cb5652a2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ab620841face4f51927aa620cb5652a2 2024-11-15T17:41:07,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/3b8d16bf68104e03b8133ae8ced9ad5e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/45a9a98e86a24719a36ea5236fc5eb52, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/5d1ebf285d4246fa83982dc597698e11, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/446ac77853be4387b0c5567059a669da, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/34184ebdc5554a24af8fdf744e41136b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/25ab2f3000184da68ad46ba77e610b20, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/92e4ad3dfa08435397383edb2984e035, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/777dffe8e59a4714a3ff1d8bd90639fe, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/f3d9051d0a3e4a16a76660c1bf628719, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fd5a2e748fd74c118be8606d2dcb93e4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/842330119ae64594bc8b0a9f6bcf4a93, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/09263af7edd4457bb567fc0141f29b4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a3018de4a32d4ee48834dd2c59a7ed50, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/6f18f4ec00e44a879a66398948ad299d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d6505d0679dc4842b236077f700cfe33, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/cb2dba5339e14852a982789caebbbf80, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fabd7d0135b6433f872be249c54bb4ce, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d8bcaf0e56cc4459acc6e2996120af8a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/e0ca6d6cd06640d29431b74076a6bd47, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/ce1dd3f13b784b5b952bd6df6b59ffc9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/187e74f735c442db890d75c15d1dc0a5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/c05e460d5e654b28b9ef9af2f3876ca6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/7d9535c238f4424d86491d22ab8f9a39, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/636966ed05054cd0bacc29043c63cc22, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/100490393ef34fb09c9837f2c9a8fcae, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/9d55c08e3b104e5f91da3d64c3bf738f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/67f20883a707428f82dc64ba7a8a2fea, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/060cb40591cc4e23b21f5adaa6ac7638, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/845e46de090845eebf42f8dac22ac780, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/59a778aa0868452ba1cf32b02a5dec84] to archive 2024-11-15T17:41:07,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:41:07,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/3b8d16bf68104e03b8133ae8ced9ad5e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/3b8d16bf68104e03b8133ae8ced9ad5e 2024-11-15T17:41:07,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/45a9a98e86a24719a36ea5236fc5eb52 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/45a9a98e86a24719a36ea5236fc5eb52 2024-11-15T17:41:07,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/5d1ebf285d4246fa83982dc597698e11 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/5d1ebf285d4246fa83982dc597698e11 2024-11-15T17:41:07,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/446ac77853be4387b0c5567059a669da to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/446ac77853be4387b0c5567059a669da 2024-11-15T17:41:07,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/34184ebdc5554a24af8fdf744e41136b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/34184ebdc5554a24af8fdf744e41136b 2024-11-15T17:41:07,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/25ab2f3000184da68ad46ba77e610b20 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/25ab2f3000184da68ad46ba77e610b20 2024-11-15T17:41:07,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/92e4ad3dfa08435397383edb2984e035 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/92e4ad3dfa08435397383edb2984e035 2024-11-15T17:41:07,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/777dffe8e59a4714a3ff1d8bd90639fe to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/777dffe8e59a4714a3ff1d8bd90639fe 2024-11-15T17:41:07,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/f3d9051d0a3e4a16a76660c1bf628719 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/f3d9051d0a3e4a16a76660c1bf628719 2024-11-15T17:41:07,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fd5a2e748fd74c118be8606d2dcb93e4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fd5a2e748fd74c118be8606d2dcb93e4 2024-11-15T17:41:07,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/842330119ae64594bc8b0a9f6bcf4a93 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/842330119ae64594bc8b0a9f6bcf4a93 2024-11-15T17:41:07,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/09263af7edd4457bb567fc0141f29b4e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/09263af7edd4457bb567fc0141f29b4e 2024-11-15T17:41:07,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a3018de4a32d4ee48834dd2c59a7ed50 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a3018de4a32d4ee48834dd2c59a7ed50 2024-11-15T17:41:07,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/6f18f4ec00e44a879a66398948ad299d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/6f18f4ec00e44a879a66398948ad299d 2024-11-15T17:41:07,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d6505d0679dc4842b236077f700cfe33 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d6505d0679dc4842b236077f700cfe33 2024-11-15T17:41:07,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/cb2dba5339e14852a982789caebbbf80 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/cb2dba5339e14852a982789caebbbf80 2024-11-15T17:41:07,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fabd7d0135b6433f872be249c54bb4ce to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/fabd7d0135b6433f872be249c54bb4ce 2024-11-15T17:41:07,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d8bcaf0e56cc4459acc6e2996120af8a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/d8bcaf0e56cc4459acc6e2996120af8a 2024-11-15T17:41:07,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/e0ca6d6cd06640d29431b74076a6bd47 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/e0ca6d6cd06640d29431b74076a6bd47 2024-11-15T17:41:07,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/ce1dd3f13b784b5b952bd6df6b59ffc9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/ce1dd3f13b784b5b952bd6df6b59ffc9 2024-11-15T17:41:07,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/187e74f735c442db890d75c15d1dc0a5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/187e74f735c442db890d75c15d1dc0a5 2024-11-15T17:41:07,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/c05e460d5e654b28b9ef9af2f3876ca6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/c05e460d5e654b28b9ef9af2f3876ca6 2024-11-15T17:41:07,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/7d9535c238f4424d86491d22ab8f9a39 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/7d9535c238f4424d86491d22ab8f9a39 2024-11-15T17:41:07,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/636966ed05054cd0bacc29043c63cc22 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/636966ed05054cd0bacc29043c63cc22 2024-11-15T17:41:07,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/100490393ef34fb09c9837f2c9a8fcae to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/100490393ef34fb09c9837f2c9a8fcae 2024-11-15T17:41:07,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/9d55c08e3b104e5f91da3d64c3bf738f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/9d55c08e3b104e5f91da3d64c3bf738f 2024-11-15T17:41:07,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/67f20883a707428f82dc64ba7a8a2fea to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/67f20883a707428f82dc64ba7a8a2fea 2024-11-15T17:41:07,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/060cb40591cc4e23b21f5adaa6ac7638 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/060cb40591cc4e23b21f5adaa6ac7638 2024-11-15T17:41:07,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/845e46de090845eebf42f8dac22ac780 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/845e46de090845eebf42f8dac22ac780 2024-11-15T17:41:07,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/59a778aa0868452ba1cf32b02a5dec84 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/59a778aa0868452ba1cf32b02a5dec84 2024-11-15T17:41:07,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2580a64296c54d8186149fcb4ae5fecf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7367b4e79b054a60b4fc240783d3636a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1f66670625794b07bbcbe012a26ba7a2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/578f22cd946f4ea4893f006fd095cbac, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/5e7b174fc1564a8ca1fb642253ad0594, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/f8005dafeee34be3b6b03ce7548e1202, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/d67c0498dab14fc4989cbdd8f49369c3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/14a0c768864346b99a3725dc6f7691cb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2b3b7ab89fe44d3eb7a734218d29ded7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b69f3bd089a3489b9bdbad1e068799f1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/440e2c7a735646298117d64458967812, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b6b31686cc614fa780d788003e17f7fa, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/c138e82d4fce4edea13db8ba213b2daf, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/70144e03173c4930a810f4ab6b8a627e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1b1b9913769b47fdb1ae2a36b2649d30, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/cf9d2dfe395f439aa41b2b6c1c9ff45d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/33ad0763fe6842eeb3c161c3c7df12a0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/57d4c57ebd004b6d85959d70697019e6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0cc9f986249c4e4f8bc7fde8dc2593c2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/49412d85528d49bc9377555a932a2097, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/91410f8f9aaa48bbb8966d9fb0e19a5e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7d3bb7d8f84942238608e40a729c10d1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/32d52134896e4ada96a65be6354c5567, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/38719834736946d7a7d2eb1585ac1e82, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/151f23da4a664bc8824297b352fa0756, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/6787474603e84aeab435319d087b2348, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0e689088f3cb4a91bc532db71836ac5a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/244fceef8f8e47528a072d75e5c81a07] to archive 2024-11-15T17:41:07,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:41:07,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2580a64296c54d8186149fcb4ae5fecf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2580a64296c54d8186149fcb4ae5fecf 2024-11-15T17:41:07,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7367b4e79b054a60b4fc240783d3636a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7367b4e79b054a60b4fc240783d3636a 2024-11-15T17:41:07,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1f66670625794b07bbcbe012a26ba7a2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1f66670625794b07bbcbe012a26ba7a2 2024-11-15T17:41:07,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/578f22cd946f4ea4893f006fd095cbac to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/578f22cd946f4ea4893f006fd095cbac 2024-11-15T17:41:07,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/5e7b174fc1564a8ca1fb642253ad0594 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/5e7b174fc1564a8ca1fb642253ad0594 2024-11-15T17:41:07,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/f8005dafeee34be3b6b03ce7548e1202 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/f8005dafeee34be3b6b03ce7548e1202 2024-11-15T17:41:07,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/d67c0498dab14fc4989cbdd8f49369c3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/d67c0498dab14fc4989cbdd8f49369c3 2024-11-15T17:41:07,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/14a0c768864346b99a3725dc6f7691cb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/14a0c768864346b99a3725dc6f7691cb 2024-11-15T17:41:07,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2b3b7ab89fe44d3eb7a734218d29ded7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/2b3b7ab89fe44d3eb7a734218d29ded7 2024-11-15T17:41:07,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b69f3bd089a3489b9bdbad1e068799f1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b69f3bd089a3489b9bdbad1e068799f1 2024-11-15T17:41:07,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/440e2c7a735646298117d64458967812 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/440e2c7a735646298117d64458967812 2024-11-15T17:41:07,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b6b31686cc614fa780d788003e17f7fa to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/b6b31686cc614fa780d788003e17f7fa 2024-11-15T17:41:07,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/c138e82d4fce4edea13db8ba213b2daf to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/c138e82d4fce4edea13db8ba213b2daf 2024-11-15T17:41:07,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/70144e03173c4930a810f4ab6b8a627e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/70144e03173c4930a810f4ab6b8a627e 2024-11-15T17:41:07,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1b1b9913769b47fdb1ae2a36b2649d30 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/1b1b9913769b47fdb1ae2a36b2649d30 2024-11-15T17:41:07,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/cf9d2dfe395f439aa41b2b6c1c9ff45d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/cf9d2dfe395f439aa41b2b6c1c9ff45d 2024-11-15T17:41:07,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/33ad0763fe6842eeb3c161c3c7df12a0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/33ad0763fe6842eeb3c161c3c7df12a0 2024-11-15T17:41:07,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/57d4c57ebd004b6d85959d70697019e6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/57d4c57ebd004b6d85959d70697019e6 2024-11-15T17:41:07,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0cc9f986249c4e4f8bc7fde8dc2593c2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0cc9f986249c4e4f8bc7fde8dc2593c2 2024-11-15T17:41:07,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/49412d85528d49bc9377555a932a2097 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/49412d85528d49bc9377555a932a2097 2024-11-15T17:41:07,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/91410f8f9aaa48bbb8966d9fb0e19a5e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/91410f8f9aaa48bbb8966d9fb0e19a5e 2024-11-15T17:41:07,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7d3bb7d8f84942238608e40a729c10d1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/7d3bb7d8f84942238608e40a729c10d1 2024-11-15T17:41:07,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/32d52134896e4ada96a65be6354c5567 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/32d52134896e4ada96a65be6354c5567 2024-11-15T17:41:07,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/38719834736946d7a7d2eb1585ac1e82 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/38719834736946d7a7d2eb1585ac1e82 2024-11-15T17:41:07,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/151f23da4a664bc8824297b352fa0756 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/151f23da4a664bc8824297b352fa0756 2024-11-15T17:41:07,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/6787474603e84aeab435319d087b2348 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/6787474603e84aeab435319d087b2348 2024-11-15T17:41:07,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0e689088f3cb4a91bc532db71836ac5a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/0e689088f3cb4a91bc532db71836ac5a 2024-11-15T17:41:07,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/244fceef8f8e47528a072d75e5c81a07 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/244fceef8f8e47528a072d75e5c81a07 2024-11-15T17:41:07,664 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/recovered.edits/453.seqid, newMaxSeqId=453, maxSeqId=1 2024-11-15T17:41:07,665 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. 2024-11-15T17:41:07,665 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 96ad00ac4dbe2b71b39f7138f6d0e503: Waiting for close lock at 1731692466320Running coprocessor pre-close hooks at 1731692466320Disabling compacts and flushes for region at 1731692466320Disabling writes for close at 1731692466320Obtaining lock to block concurrent updates at 1731692466320Preparing flush snapshotting stores in 96ad00ac4dbe2b71b39f7138f6d0e503 at 1731692466320Finished memstore snapshotting TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731692466321 (+1 ms)Flushing stores of TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503. at 1731692466322 (+1 ms)Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/A: creating writer at 1731692466322Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/A: appending metadata at 1731692466328 (+6 ms)Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/A: closing flushed file at 1731692466328Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/B: creating writer at 1731692466738 (+410 ms)Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/B: appending metadata at 1731692466741 (+3 ms)Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/B: closing flushed file at 1731692466741Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/C: creating writer at 1731692467156 (+415 ms)Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/C: appending metadata at 1731692467159 (+3 ms)Flushing 96ad00ac4dbe2b71b39f7138f6d0e503/C: closing flushed file at 1731692467159Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73b4869d: reopening flushed file at 1731692467568 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4df605: reopening flushed file at 1731692467573 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71683718: reopening flushed file at 1731692467577 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 96ad00ac4dbe2b71b39f7138f6d0e503 in 1261ms, sequenceid=450, compaction requested=true at 1731692467581 (+4 ms)Writing region close event to WAL at 1731692467662 (+81 ms)Running coprocessor post-close hooks at 1731692467665 (+3 ms)Closed at 1731692467665 2024-11-15T17:41:07,666 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:41:07,667 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=96ad00ac4dbe2b71b39f7138f6d0e503, regionState=CLOSED 2024-11-15T17:41:07,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:41:07,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T17:41:07,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 96ad00ac4dbe2b71b39f7138f6d0e503, server=4e3b17656000,45187,1731692383005 in 1.5020 sec 2024-11-15T17:41:07,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-15T17:41:07,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96ad00ac4dbe2b71b39f7138f6d0e503, UNASSIGN in 1.5060 sec 2024-11-15T17:41:07,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-15T17:41:07,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-15T17:41:07,674 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692467674"}]},"ts":"1731692467674"} 2024-11-15T17:41:07,675 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:41:07,675 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:41:07,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-15T17:41:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T17:41:08,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:08,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:41:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:08,317 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T17:41:08,318 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:08,321 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:41:08,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/recovered.edits] 2024-11-15T17:41:08,325 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cd3217d548814be58ffbbfeb120ae2e6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/cd3217d548814be58ffbbfeb120ae2e6 2024-11-15T17:41:08,326 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ce4c6456046c4c21a9861905007b312c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/ce4c6456046c4c21a9861905007b312c 2024-11-15T17:41:08,327 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f4c6e52325d8445e97e2becbee784564 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/A/f4c6e52325d8445e97e2becbee784564 2024-11-15T17:41:08,328 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/74cfb151802f4405a73122fc5f587bdc to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/74cfb151802f4405a73122fc5f587bdc 2024-11-15T17:41:08,329 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a71c6e920f7d4341a925b4972dcff53a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/a71c6e920f7d4341a925b4972dcff53a 2024-11-15T17:41:08,330 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/de667f50e06e4b419049f3882b6c60a5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/B/de667f50e06e4b419049f3882b6c60a5 2024-11-15T17:41:08,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/4c17c933d4334538b37d98b118317157 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/4c17c933d4334538b37d98b118317157 2024-11-15T17:41:08,333 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a1947ba70ef241cda167dea6163d174f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a1947ba70ef241cda167dea6163d174f 2024-11-15T17:41:08,334 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a3df1141866c4ea6981011224aff3c14 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/a3df1141866c4ea6981011224aff3c14 2024-11-15T17:41:08,335 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/e4617c7db95a4a7c8baae0e2549a6865 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/C/e4617c7db95a4a7c8baae0e2549a6865 2024-11-15T17:41:08,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/recovered.edits/453.seqid to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503/recovered.edits/453.seqid 2024-11-15T17:41:08,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/96ad00ac4dbe2b71b39f7138f6d0e503 2024-11-15T17:41:08,338 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:41:08,340 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:08,343 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:41:08,345 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:41:08,347 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:08,347 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:41:08,347 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731692468347"}]},"ts":"9223372036854775807"} 2024-11-15T17:41:08,349 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:41:08,349 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 96ad00ac4dbe2b71b39f7138f6d0e503, NAME => 'TestAcidGuarantees,,1731692439553.96ad00ac4dbe2b71b39f7138f6d0e503.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:41:08,349 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:41:08,350 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731692468349"}]},"ts":"9223372036854775807"} 2024-11-15T17:41:08,352 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:41:08,352 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:08,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-15T17:41:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T17:41:08,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:41:08,428 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:08,434 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 247), OpenFileDescriptor=455 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=461 (was 532), ProcessCount=11 (was 11), AvailableMemoryMB=8947 (was 9458) 2024-11-15T17:41:08,444 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=461, ProcessCount=11, AvailableMemoryMB=8947 2024-11-15T17:41:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:41:08,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:41:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:08,448 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:41:08,448 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:08,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-15T17:41:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T17:41:08,449 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:41:08,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742165_1341 (size=959) 2024-11-15T17:41:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T17:41:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T17:41:08,857 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:41:08,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742166_1342 (size=53) 2024-11-15T17:41:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T17:41:09,297 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:41:09,297 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 057f1b8c8fc0614d025e1dac4cdaab16, disabling compactions & flushes 2024-11-15T17:41:09,297 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:09,297 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:09,297 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. after waiting 0 ms 2024-11-15T17:41:09,297 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:09,297 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:09,297 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 057f1b8c8fc0614d025e1dac4cdaab16: Waiting for close lock at 1731692469297Disabling compacts and flushes for region at 1731692469297Disabling writes for close at 1731692469297Writing region close event to WAL at 1731692469297Closed at 1731692469297 2024-11-15T17:41:09,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:41:09,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731692469299"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692469299"}]},"ts":"1731692469299"} 2024-11-15T17:41:09,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:41:09,303 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:41:09,303 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692469303"}]},"ts":"1731692469303"} 2024-11-15T17:41:09,304 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T17:41:09,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, ASSIGN}] 2024-11-15T17:41:09,305 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, ASSIGN 2024-11-15T17:41:09,306 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, ASSIGN; state=OFFLINE, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=false 2024-11-15T17:41:09,457 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:09,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, ASSIGN because future has completed 2024-11-15T17:41:09,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:41:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T17:41:09,618 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:09,618 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:41:09,618 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,619 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:41:09,619 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,619 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,620 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,621 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057f1b8c8fc0614d025e1dac4cdaab16 columnFamilyName A 2024-11-15T17:41:09,621 DEBUG [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:09,622 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(327): Store=057f1b8c8fc0614d025e1dac4cdaab16/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:09,622 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,623 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057f1b8c8fc0614d025e1dac4cdaab16 columnFamilyName B 2024-11-15T17:41:09,623 DEBUG [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:09,623 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(327): Store=057f1b8c8fc0614d025e1dac4cdaab16/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:09,624 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,625 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057f1b8c8fc0614d025e1dac4cdaab16 columnFamilyName C 2024-11-15T17:41:09,625 DEBUG [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:09,625 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(327): Store=057f1b8c8fc0614d025e1dac4cdaab16/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:09,625 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,626 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,626 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,627 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,627 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,627 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:41:09,628 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,630 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:41:09,630 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 057f1b8c8fc0614d025e1dac4cdaab16; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67919485, jitterRate=0.012079194188117981}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:41:09,630 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:09,631 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 057f1b8c8fc0614d025e1dac4cdaab16: Running coprocessor pre-open hook at 1731692469619Writing region info on filesystem at 1731692469619Initializing all the Stores at 1731692469619Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692469619Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692469620 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692469620Cleaning up temporary data from old regions at 1731692469627 (+7 ms)Running coprocessor post-open hooks at 1731692469630 (+3 ms)Region opened successfully at 1731692469631 (+1 ms) 2024-11-15T17:41:09,632 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., pid=93, masterSystemTime=1731692469615 2024-11-15T17:41:09,633 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:09,633 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:09,634 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=OPEN, openSeqNum=2, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:09,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:41:09,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-15T17:41:09,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 in 175 msec 2024-11-15T17:41:09,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-15T17:41:09,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, ASSIGN in 333 msec 2024-11-15T17:41:09,640 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:41:09,640 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692469640"}]},"ts":"1731692469640"} 2024-11-15T17:41:09,641 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T17:41:09,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:41:09,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1970 sec 2024-11-15T17:41:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T17:41:10,595 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:10,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:10,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:10,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:10,597 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9dad0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:10,599 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b5b603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:10,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:10,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:10,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:10,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:41:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4e3b17656000,41205,1731692382118 2024-11-15T17:41:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46d2bffd 2024-11-15T17:41:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:41:10,603 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:41:10,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:41:10,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:41:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:10,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742167_1343 (size=995) 2024-11-15T17:41:11,015 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T17:41:11,015 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T17:41:11,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:41:11,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, REOPEN/MOVE}] 2024-11-15T17:41:11,021 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, REOPEN/MOVE 2024-11-15T17:41:11,022 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, REOPEN/MOVE because future has completed 2024-11-15T17:41:11,024 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:41:11,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:41:11,177 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,177 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:41:11,177 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 057f1b8c8fc0614d025e1dac4cdaab16, disabling compactions & flushes 2024-11-15T17:41:11,177 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,177 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,177 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. after waiting 0 ms 2024-11-15T17:41:11,177 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,186 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T17:41:11,187 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,187 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 057f1b8c8fc0614d025e1dac4cdaab16: Waiting for close lock at 1731692471177Running coprocessor pre-close hooks at 1731692471177Disabling compacts and flushes for region at 1731692471177Disabling writes for close at 1731692471177Writing region close event to WAL at 1731692471178 (+1 ms)Running coprocessor post-close hooks at 1731692471187 (+9 ms)Closed at 1731692471187 2024-11-15T17:41:11,188 WARN [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 057f1b8c8fc0614d025e1dac4cdaab16 to self. 2024-11-15T17:41:11,191 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,192 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=CLOSED 2024-11-15T17:41:11,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:41:11,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-15T17:41:11,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 in 171 msec 2024-11-15T17:41:11,199 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, REOPEN/MOVE; state=CLOSED, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=true 2024-11-15T17:41:11,235 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:41:11,350 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, REOPEN/MOVE because future has completed 2024-11-15T17:41:11,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:41:11,510 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,511 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:41:11,511 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,511 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:41:11,512 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,512 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,514 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,516 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057f1b8c8fc0614d025e1dac4cdaab16 columnFamilyName A 2024-11-15T17:41:11,518 DEBUG [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:11,519 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(327): Store=057f1b8c8fc0614d025e1dac4cdaab16/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:11,519 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,520 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057f1b8c8fc0614d025e1dac4cdaab16 columnFamilyName B 2024-11-15T17:41:11,520 DEBUG [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:11,521 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(327): Store=057f1b8c8fc0614d025e1dac4cdaab16/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:11,521 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,522 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057f1b8c8fc0614d025e1dac4cdaab16 columnFamilyName C 2024-11-15T17:41:11,522 DEBUG [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:11,523 INFO [StoreOpener-057f1b8c8fc0614d025e1dac4cdaab16-1 {}] regionserver.HStore(327): Store=057f1b8c8fc0614d025e1dac4cdaab16/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:11,523 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,524 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,525 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,527 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,527 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,528 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:41:11,530 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,531 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 057f1b8c8fc0614d025e1dac4cdaab16; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68547188, jitterRate=0.021432697772979736}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:41:11,531 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,532 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 057f1b8c8fc0614d025e1dac4cdaab16: Running coprocessor pre-open hook at 1731692471512Writing region info on filesystem at 1731692471512Initializing all the Stores at 1731692471513 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692471514 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692471514Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692471514Cleaning up temporary data from old regions at 1731692471527 (+13 ms)Running coprocessor post-open hooks at 1731692471531 (+4 ms)Region opened successfully at 1731692471532 (+1 ms) 2024-11-15T17:41:11,533 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., pid=98, masterSystemTime=1731692471505 2024-11-15T17:41:11,536 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,536 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,536 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=OPEN, openSeqNum=5, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:41:11,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-15T17:41:11,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 in 187 msec 2024-11-15T17:41:11,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T17:41:11,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, REOPEN/MOVE in 522 msec 2024-11-15T17:41:11,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-15T17:41:11,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-15T17:41:11,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-11-15T17:41:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-15T17:41:11,552 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:11,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,553 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7548d1de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,555 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67eaa221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,557 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724f7dea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,559 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc77179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,562 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b6dcf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,563 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0bc8a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,564 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85acfb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,566 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449a4901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,567 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f036c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,568 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6298e0b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,570 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5911c7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,571 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fb98c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,573 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1067ed0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,574 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba8863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,576 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64432f7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,577 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0c6c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,579 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8d949a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@937b4f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:11,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:11,582 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4c05b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:11,583 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36141803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:11,584 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,584 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,584 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,584 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,584 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,585 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,585 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,585 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:11,585 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,586 DEBUG [Thread-1574 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,586 DEBUG [Thread-1574 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,586 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-15T17:41:11,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,587 DEBUG [Thread-1565 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,587 DEBUG [Thread-1565 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,588 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:11,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T17:41:11,589 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:11,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,591 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:11,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:11,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:41:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:11,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:41:11,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111565c190f3e8b14911ad9f8c6bbe5c41a2_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692471609/Put/seqid=0 2024-11-15T17:41:11,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692531650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692531654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692531654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692531654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692531655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742168_1344 (size=14594) 2024-11-15T17:41:11,676 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:11,684 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111565c190f3e8b14911ad9f8c6bbe5c41a2_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111565c190f3e8b14911ad9f8c6bbe5c41a2_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:11,686 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1521e9ffbfd547579733a8d6c8190cbe, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:11,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1521e9ffbfd547579733a8d6c8190cbe is 175, key is test_row_0/A:col10/1731692471609/Put/seqid=0 2024-11-15T17:41:11,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742169_1345 (size=39549) 2024-11-15T17:41:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T17:41:11,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:11,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692531765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692531775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692531775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692531776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692531776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:11,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:11,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T17:41:11,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692531975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692531986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692531986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692531986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692531986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1521e9ffbfd547579733a8d6c8190cbe 2024-11-15T17:41:12,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/04c25b8026894357b2f1001270cc1ea8 is 50, key is test_row_0/B:col10/1731692471609/Put/seqid=0 2024-11-15T17:41:12,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742170_1346 (size=12001) 2024-11-15T17:41:12,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T17:41:12,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692532285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692532296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692532296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692532296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692532296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:12,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:12,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/04c25b8026894357b2f1001270cc1ea8 2024-11-15T17:41:12,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/48b8af021b5849c987cdfa138a9e35d0 is 50, key is test_row_0/C:col10/1731692471609/Put/seqid=0 2024-11-15T17:41:12,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742171_1347 (size=12001) 2024-11-15T17:41:12,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:12,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:12,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T17:41:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692532796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692532806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692532806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692532806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:12,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692532815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:12,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/48b8af021b5849c987cdfa138a9e35d0 2024-11-15T17:41:12,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1521e9ffbfd547579733a8d6c8190cbe as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1521e9ffbfd547579733a8d6c8190cbe 2024-11-15T17:41:12,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1521e9ffbfd547579733a8d6c8190cbe, entries=200, sequenceid=16, filesize=38.6 K 2024-11-15T17:41:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/04c25b8026894357b2f1001270cc1ea8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/04c25b8026894357b2f1001270cc1ea8 2024-11-15T17:41:12,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/04c25b8026894357b2f1001270cc1ea8, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T17:41:12,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/48b8af021b5849c987cdfa138a9e35d0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/48b8af021b5849c987cdfa138a9e35d0 2024-11-15T17:41:12,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/48b8af021b5849c987cdfa138a9e35d0, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T17:41:12,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1353ms, sequenceid=16, compaction requested=false 2024-11-15T17:41:12,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:12,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T17:41:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:12,964 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:41:12,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150342e8a7d1d24df49f7586db8288281a_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692471651/Put/seqid=0 2024-11-15T17:41:12,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742172_1348 (size=12154) 2024-11-15T17:41:13,196 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:41:13,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:13,380 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150342e8a7d1d24df49f7586db8288281a_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150342e8a7d1d24df49f7586db8288281a_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1e2d879051114a7da44969aee7204308, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:13,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1e2d879051114a7da44969aee7204308 is 175, key is test_row_0/A:col10/1731692471651/Put/seqid=0 2024-11-15T17:41:13,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742173_1349 (size=30955) 2024-11-15T17:41:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T17:41:13,803 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1e2d879051114a7da44969aee7204308 2024-11-15T17:41:13,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/743020fe649a4407a26b22678b4e6588 is 50, key is test_row_0/B:col10/1731692471651/Put/seqid=0 2024-11-15T17:41:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:13,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:13,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742174_1350 (size=12001) 2024-11-15T17:41:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692533836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692533840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,841 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692533840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692533841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692533842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692533946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692533955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692533955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692533955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692534156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692534165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692534165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692534165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,227 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/743020fe649a4407a26b22678b4e6588 2024-11-15T17:41:14,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/a440d91684bf450185345e6f4d4c4d50 is 50, key is test_row_0/C:col10/1731692471651/Put/seqid=0 2024-11-15T17:41:14,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742175_1351 (size=12001) 2024-11-15T17:41:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692534465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692534476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692534476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692534485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:14,652 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/a440d91684bf450185345e6f4d4c4d50 2024-11-15T17:41:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1e2d879051114a7da44969aee7204308 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1e2d879051114a7da44969aee7204308 2024-11-15T17:41:14,666 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1e2d879051114a7da44969aee7204308, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T17:41:14,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/743020fe649a4407a26b22678b4e6588 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/743020fe649a4407a26b22678b4e6588 2024-11-15T17:41:14,670 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/743020fe649a4407a26b22678b4e6588, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T17:41:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/a440d91684bf450185345e6f4d4c4d50 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a440d91684bf450185345e6f4d4c4d50 2024-11-15T17:41:14,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a440d91684bf450185345e6f4d4c4d50, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T17:41:14,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1710ms, sequenceid=40, compaction requested=false 2024-11-15T17:41:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-15T17:41:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-15T17:41:14,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T17:41:14,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0870 sec 2024-11-15T17:41:14,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.0930 sec 2024-11-15T17:41:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:14,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:14,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:41:14,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e606ba8b95f041f997145f1e83b5b129_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692473839/Put/seqid=0 2024-11-15T17:41:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742176_1352 (size=14594) 2024-11-15T17:41:15,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:15,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e606ba8b95f041f997145f1e83b5b129_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e606ba8b95f041f997145f1e83b5b129_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:15,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7e3e63ce283f451d9c93e49ea1ef9928, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:15,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7e3e63ce283f451d9c93e49ea1ef9928 is 175, key is test_row_0/A:col10/1731692473839/Put/seqid=0 2024-11-15T17:41:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742177_1353 (size=39549) 2024-11-15T17:41:15,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692535058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692535064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692535068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692535068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692535176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692535176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692535176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692535176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692535386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692535386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692535386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692535395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,408 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7e3e63ce283f451d9c93e49ea1ef9928 2024-11-15T17:41:15,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/2b4c65f16045412db06caa9ba0c7672d is 50, key is test_row_0/B:col10/1731692473839/Put/seqid=0 2024-11-15T17:41:15,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742178_1354 (size=12001) 2024-11-15T17:41:15,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692535695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692535706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692535706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692535715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T17:41:15,756 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:15,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T17:41:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T17:41:15,758 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:15,758 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:15,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:15,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/2b4c65f16045412db06caa9ba0c7672d 2024-11-15T17:41:15,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/3b6aedda3869472bb5acc1936e494fb0 is 50, key is test_row_0/C:col10/1731692473839/Put/seqid=0 2024-11-15T17:41:15,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742179_1355 (size=12001) 2024-11-15T17:41:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T17:41:15,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692535865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:15,872 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:15,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T17:41:15,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:15,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:16,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T17:41:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T17:41:16,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T17:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:16,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692536215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692536225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692536226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692536226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:16,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/3b6aedda3869472bb5acc1936e494fb0 2024-11-15T17:41:16,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7e3e63ce283f451d9c93e49ea1ef9928 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7e3e63ce283f451d9c93e49ea1ef9928 2024-11-15T17:41:16,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7e3e63ce283f451d9c93e49ea1ef9928, entries=200, sequenceid=53, filesize=38.6 K 2024-11-15T17:41:16,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/2b4c65f16045412db06caa9ba0c7672d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/2b4c65f16045412db06caa9ba0c7672d 2024-11-15T17:41:16,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/2b4c65f16045412db06caa9ba0c7672d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T17:41:16,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/3b6aedda3869472bb5acc1936e494fb0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3b6aedda3869472bb5acc1936e494fb0 2024-11-15T17:41:16,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3b6aedda3869472bb5acc1936e494fb0, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T17:41:16,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1270ms, sequenceid=53, compaction requested=true 2024-11-15T17:41:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:16,258 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:16,258 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:16,259 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:16,259 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:16,259 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,259 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1521e9ffbfd547579733a8d6c8190cbe, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1e2d879051114a7da44969aee7204308, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7e3e63ce283f451d9c93e49ea1ef9928] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=107.5 K 2024-11-15T17:41:16,259 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,259 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1521e9ffbfd547579733a8d6c8190cbe, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1e2d879051114a7da44969aee7204308, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7e3e63ce283f451d9c93e49ea1ef9928] 2024-11-15T17:41:16,259 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:16,259 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:16,259 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,259 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/04c25b8026894357b2f1001270cc1ea8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/743020fe649a4407a26b22678b4e6588, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/2b4c65f16045412db06caa9ba0c7672d] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=35.2 K 2024-11-15T17:41:16,259 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1521e9ffbfd547579733a8d6c8190cbe, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692471601 2024-11-15T17:41:16,260 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e2d879051114a7da44969aee7204308, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731692471641 2024-11-15T17:41:16,260 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 04c25b8026894357b2f1001270cc1ea8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692471608 2024-11-15T17:41:16,260 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e3e63ce283f451d9c93e49ea1ef9928, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731692473829 2024-11-15T17:41:16,260 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 743020fe649a4407a26b22678b4e6588, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731692471641 2024-11-15T17:41:16,260 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b4c65f16045412db06caa9ba0c7672d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731692473839 2024-11-15T17:41:16,265 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:16,265 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/36ae0aa2b41249fb83130f7e4d15d237 is 50, key is test_row_0/B:col10/1731692473839/Put/seqid=0 2024-11-15T17:41:16,266 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:16,268 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b17c0517b7e846aab50846b8c1326341_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:16,270 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b17c0517b7e846aab50846b8c1326341_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:16,271 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b17c0517b7e846aab50846b8c1326341_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:16,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742180_1356 (size=12104) 2024-11-15T17:41:16,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742181_1357 (size=4469) 2024-11-15T17:41:16,276 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#302 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:16,276 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/50040307041b4cc69aadaceec595cb5f is 175, key is test_row_0/A:col10/1731692473839/Put/seqid=0 2024-11-15T17:41:16,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742182_1358 (size=31058) 2024-11-15T17:41:16,285 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/50040307041b4cc69aadaceec595cb5f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/50040307041b4cc69aadaceec595cb5f 2024-11-15T17:41:16,292 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into 50040307041b4cc69aadaceec595cb5f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:16,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:16,292 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692476258; duration=0sec 2024-11-15T17:41:16,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:16,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:16,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:16,293 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:16,293 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:16,293 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,293 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/48b8af021b5849c987cdfa138a9e35d0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a440d91684bf450185345e6f4d4c4d50, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3b6aedda3869472bb5acc1936e494fb0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=35.2 K 2024-11-15T17:41:16,293 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b8af021b5849c987cdfa138a9e35d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692471608 2024-11-15T17:41:16,294 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting a440d91684bf450185345e6f4d4c4d50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731692471641 2024-11-15T17:41:16,294 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b6aedda3869472bb5acc1936e494fb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731692473839 2024-11-15T17:41:16,302 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:16,302 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c15b7f7a9b3c41b5a3f5f5e5aa762df2 is 50, key is test_row_0/C:col10/1731692473839/Put/seqid=0 2024-11-15T17:41:16,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742183_1359 (size=12104) 2024-11-15T17:41:16,314 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c15b7f7a9b3c41b5a3f5f5e5aa762df2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c15b7f7a9b3c41b5a3f5f5e5aa762df2 2024-11-15T17:41:16,323 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into c15b7f7a9b3c41b5a3f5f5e5aa762df2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:16,323 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:16,323 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=13, startTime=1731692476258; duration=0sec 2024-11-15T17:41:16,323 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:16,323 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:16,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T17:41:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:16,369 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:41:16,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115254af4c85ffd4acca32a625f1e5496f1_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692475057/Put/seqid=0 2024-11-15T17:41:16,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742184_1360 (size=12154) 2024-11-15T17:41:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T17:41:16,676 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/36ae0aa2b41249fb83130f7e4d15d237 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/36ae0aa2b41249fb83130f7e4d15d237 2024-11-15T17:41:16,680 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 36ae0aa2b41249fb83130f7e4d15d237(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:16,680 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:16,680 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692476258; duration=0sec 2024-11-15T17:41:16,680 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:16,680 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:16,782 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115254af4c85ffd4acca32a625f1e5496f1_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115254af4c85ffd4acca32a625f1e5496f1_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:16,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/590ec6ef1ce74ce583575c98091b3232, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/590ec6ef1ce74ce583575c98091b3232 is 175, key is test_row_0/A:col10/1731692475057/Put/seqid=0 2024-11-15T17:41:16,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742185_1361 (size=30955) 2024-11-15T17:41:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T17:41:17,187 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/590ec6ef1ce74ce583575c98091b3232 2024-11-15T17:41:17,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f6cf921b043140b3aa1c77d6269aa782 is 50, key is test_row_0/B:col10/1731692475057/Put/seqid=0 2024-11-15T17:41:17,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742186_1362 (size=12001) 2024-11-15T17:41:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:17,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692537251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692537251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692537252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692537258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692537366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692537366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692537366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692537366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692537576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692537576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692537576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692537576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,598 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f6cf921b043140b3aa1c77d6269aa782 2024-11-15T17:41:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c676aa0b21e843ad87f866481a12ae10 is 50, key is test_row_0/C:col10/1731692475057/Put/seqid=0 2024-11-15T17:41:17,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742187_1363 (size=12001) 2024-11-15T17:41:17,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692537885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692537885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692537886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692537886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T17:41:18,008 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c676aa0b21e843ad87f866481a12ae10 2024-11-15T17:41:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/590ec6ef1ce74ce583575c98091b3232 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/590ec6ef1ce74ce583575c98091b3232 2024-11-15T17:41:18,015 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/590ec6ef1ce74ce583575c98091b3232, entries=150, sequenceid=78, filesize=30.2 K 2024-11-15T17:41:18,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f6cf921b043140b3aa1c77d6269aa782 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f6cf921b043140b3aa1c77d6269aa782 2024-11-15T17:41:18,019 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f6cf921b043140b3aa1c77d6269aa782, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T17:41:18,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c676aa0b21e843ad87f866481a12ae10 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c676aa0b21e843ad87f866481a12ae10 2024-11-15T17:41:18,023 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c676aa0b21e843ad87f866481a12ae10, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T17:41:18,023 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1654ms, sequenceid=78, compaction requested=false 2024-11-15T17:41:18,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:18,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:18,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T17:41:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T17:41:18,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T17:41:18,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2670 sec 2024-11-15T17:41:18,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.2710 sec 2024-11-15T17:41:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:18,399 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:18,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:41:18,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d990831a5354146acaadc4bdbfbf059_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692477257/Put/seqid=0 2024-11-15T17:41:18,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742188_1364 (size=17034) 2024-11-15T17:41:18,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692538430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692538431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692538432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692538433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692538545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692538546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692538547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692538547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692538756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692538756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692538756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692538765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:18,809 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:18,815 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d990831a5354146acaadc4bdbfbf059_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d990831a5354146acaadc4bdbfbf059_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:18,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/fc9b93741b8f4ad78f4db70b52e93137, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:18,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/fc9b93741b8f4ad78f4db70b52e93137 is 175, key is test_row_0/A:col10/1731692477257/Put/seqid=0 2024-11-15T17:41:18,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742189_1365 (size=48139) 2024-11-15T17:41:18,829 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/fc9b93741b8f4ad78f4db70b52e93137 2024-11-15T17:41:18,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/ff6ba109620d4986962ded2b535dc70e is 50, key is test_row_0/B:col10/1731692477257/Put/seqid=0 2024-11-15T17:41:18,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742190_1366 (size=12001) 2024-11-15T17:41:18,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/ff6ba109620d4986962ded2b535dc70e 2024-11-15T17:41:18,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/b66055d929444b39bd018c38ce890970 is 50, key is test_row_0/C:col10/1731692477257/Put/seqid=0 2024-11-15T17:41:18,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742191_1367 (size=12001) 2024-11-15T17:41:18,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/b66055d929444b39bd018c38ce890970 2024-11-15T17:41:18,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/fc9b93741b8f4ad78f4db70b52e93137 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/fc9b93741b8f4ad78f4db70b52e93137 2024-11-15T17:41:18,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/fc9b93741b8f4ad78f4db70b52e93137, entries=250, sequenceid=94, filesize=47.0 K 2024-11-15T17:41:18,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/ff6ba109620d4986962ded2b535dc70e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/ff6ba109620d4986962ded2b535dc70e 2024-11-15T17:41:18,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/ff6ba109620d4986962ded2b535dc70e, entries=150, sequenceid=94, filesize=11.7 K 2024-11-15T17:41:18,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/b66055d929444b39bd018c38ce890970 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/b66055d929444b39bd018c38ce890970 2024-11-15T17:41:18,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/b66055d929444b39bd018c38ce890970, entries=150, sequenceid=94, filesize=11.7 K 2024-11-15T17:41:18,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 057f1b8c8fc0614d025e1dac4cdaab16 in 489ms, sequenceid=94, compaction requested=true 2024-11-15T17:41:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:18,888 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T17:41:18,890 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:18,890 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:18,890 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:18,890 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:18,891 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/50040307041b4cc69aadaceec595cb5f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/590ec6ef1ce74ce583575c98091b3232, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/fc9b93741b8f4ad78f4db70b52e93137] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=107.6 K 2024-11-15T17:41:18,891 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/50040307041b4cc69aadaceec595cb5f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/590ec6ef1ce74ce583575c98091b3232, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/fc9b93741b8f4ad78f4db70b52e93137] 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:18,891 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50040307041b4cc69aadaceec595cb5f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731692473839 2024-11-15T17:41:18,891 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/36ae0aa2b41249fb83130f7e4d15d237, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f6cf921b043140b3aa1c77d6269aa782, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/ff6ba109620d4986962ded2b535dc70e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=35.3 K 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590ec6ef1ce74ce583575c98091b3232, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731692475057 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ae0aa2b41249fb83130f7e4d15d237, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731692473839 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc9b93741b8f4ad78f4db70b52e93137, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692477249 2024-11-15T17:41:18,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f6cf921b043140b3aa1c77d6269aa782, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731692475057 2024-11-15T17:41:18,892 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6ba109620d4986962ded2b535dc70e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692477257 2024-11-15T17:41:18,911 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:18,914 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:18,914 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/8695c0cdd23546e6a388f2a90b398052 is 50, key is test_row_0/B:col10/1731692477257/Put/seqid=0 2024-11-15T17:41:18,918 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115266d685d9b0e4313b7ead1624af72485_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:18,920 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115266d685d9b0e4313b7ead1624af72485_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:18,920 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115266d685d9b0e4313b7ead1624af72485_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:18,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742192_1368 (size=12207) 2024-11-15T17:41:18,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742193_1369 (size=4469) 2024-11-15T17:41:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:19,070 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:19,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T17:41:19,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ca31d8557cab41e699e212dba61d8577_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692478430/Put/seqid=0 2024-11-15T17:41:19,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742194_1370 (size=14594) 2024-11-15T17:41:19,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:19,095 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ca31d8557cab41e699e212dba61d8577_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ca31d8557cab41e699e212dba61d8577_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:19,096 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e4a2f169c614f79a9a606464dd2b716, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:19,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e4a2f169c614f79a9a606464dd2b716 is 175, key is test_row_0/A:col10/1731692478430/Put/seqid=0 2024-11-15T17:41:19,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742195_1371 (size=39549) 2024-11-15T17:41:19,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e4a2f169c614f79a9a606464dd2b716 2024-11-15T17:41:19,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/55cb991ee1044a5cad0b10e18c5e4d78 is 50, key is test_row_0/B:col10/1731692478430/Put/seqid=0 2024-11-15T17:41:19,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692539101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692539103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692539110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692539111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742196_1372 (size=12001) 2024-11-15T17:41:19,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/55cb991ee1044a5cad0b10e18c5e4d78 2024-11-15T17:41:19,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c227c7b0233943b1b6bc1e7aecf65d1b is 50, key is test_row_0/C:col10/1731692478430/Put/seqid=0 2024-11-15T17:41:19,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742197_1373 (size=12001) 2024-11-15T17:41:19,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c227c7b0233943b1b6bc1e7aecf65d1b 2024-11-15T17:41:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e4a2f169c614f79a9a606464dd2b716 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e4a2f169c614f79a9a606464dd2b716 2024-11-15T17:41:19,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e4a2f169c614f79a9a606464dd2b716, entries=200, sequenceid=116, filesize=38.6 K 2024-11-15T17:41:19,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/55cb991ee1044a5cad0b10e18c5e4d78 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/55cb991ee1044a5cad0b10e18c5e4d78 2024-11-15T17:41:19,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/55cb991ee1044a5cad0b10e18c5e4d78, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T17:41:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/c227c7b0233943b1b6bc1e7aecf65d1b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c227c7b0233943b1b6bc1e7aecf65d1b 2024-11-15T17:41:19,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c227c7b0233943b1b6bc1e7aecf65d1b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T17:41:19,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 057f1b8c8fc0614d025e1dac4cdaab16 in 96ms, sequenceid=116, compaction requested=true 2024-11-15T17:41:19,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:19,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:19,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T17:41:19,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:19,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T17:41:19,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:19,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T17:41:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:19,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:19,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T17:41:19,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159f8f9bb5e49d494dac43afc4d949f957_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692479109/Put/seqid=0 2024-11-15T17:41:19,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742198_1374 (size=14694) 2024-11-15T17:41:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692539282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692539282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692539282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692539283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,336 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:19,337 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/c3d62611c1fe490cb62e85846cbe914f is 175, key is test_row_0/A:col10/1731692477257/Put/seqid=0 2024-11-15T17:41:19,337 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/8695c0cdd23546e6a388f2a90b398052 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/8695c0cdd23546e6a388f2a90b398052 2024-11-15T17:41:19,343 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 8695c0cdd23546e6a388f2a90b398052(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:19,343 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:19,343 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692478889; duration=0sec 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. because compaction request was cancelled 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:19,344 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:19,346 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:19,346 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:19,346 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:19,346 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c15b7f7a9b3c41b5a3f5f5e5aa762df2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c676aa0b21e843ad87f866481a12ae10, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/b66055d929444b39bd018c38ce890970, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c227c7b0233943b1b6bc1e7aecf65d1b] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=47.0 K 2024-11-15T17:41:19,347 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c15b7f7a9b3c41b5a3f5f5e5aa762df2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731692473839 2024-11-15T17:41:19,347 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c676aa0b21e843ad87f866481a12ae10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731692475057 2024-11-15T17:41:19,347 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b66055d929444b39bd018c38ce890970, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692477257 2024-11-15T17:41:19,347 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c227c7b0233943b1b6bc1e7aecf65d1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692478430 2024-11-15T17:41:19,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742199_1375 (size=31161) 2024-11-15T17:41:19,355 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/c3d62611c1fe490cb62e85846cbe914f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/c3d62611c1fe490cb62e85846cbe914f 2024-11-15T17:41:19,355 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:19,356 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/6745bb5d09cc4227a8ef81f5bb4689ee is 50, key is test_row_0/C:col10/1731692478430/Put/seqid=0 2024-11-15T17:41:19,367 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into c3d62611c1fe490cb62e85846cbe914f(size=30.4 K), total size for store is 69.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:19,367 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:19,367 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692478888; duration=0sec 2024-11-15T17:41:19,367 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T17:41:19,367 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:19,367 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:19,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-15T17:41:19,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:41:19,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:41:19,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. because compaction request was cancelled 2024-11-15T17:41:19,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:19,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T17:41:19,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:41:19,369 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:41:19,369 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. because compaction request was cancelled 2024-11-15T17:41:19,369 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:19,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742200_1376 (size=12241) 2024-11-15T17:41:19,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692539406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692539406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692539407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692539407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692539626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692539626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692539626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692539626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,640 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:19,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159f8f9bb5e49d494dac43afc4d949f957_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f8f9bb5e49d494dac43afc4d949f957_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:19,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/54f9f6fdcc7340ad876e7842a54b29a0, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:19,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/54f9f6fdcc7340ad876e7842a54b29a0 is 175, key is test_row_0/A:col10/1731692479109/Put/seqid=0 2024-11-15T17:41:19,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742201_1377 (size=39649) 2024-11-15T17:41:19,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/54f9f6fdcc7340ad876e7842a54b29a0 2024-11-15T17:41:19,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/fba3115320804120a2e84c831db2299d is 50, key is test_row_0/B:col10/1731692479109/Put/seqid=0 2024-11-15T17:41:19,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742202_1378 (size=12101) 2024-11-15T17:41:19,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/fba3115320804120a2e84c831db2299d 2024-11-15T17:41:19,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/24276baec9914b558f88d43846cf6049 is 50, key is test_row_0/C:col10/1731692479109/Put/seqid=0 2024-11-15T17:41:19,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742203_1379 (size=12101) 2024-11-15T17:41:19,791 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/6745bb5d09cc4227a8ef81f5bb4689ee as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6745bb5d09cc4227a8ef81f5bb4689ee 2024-11-15T17:41:19,794 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into 6745bb5d09cc4227a8ef81f5bb4689ee(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:19,794 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:19,794 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=12, startTime=1731692479167; duration=0sec 2024-11-15T17:41:19,794 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:19,794 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692539905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,911 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T17:41:19,926 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:19,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T17:41:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T17:41:19,928 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:19,929 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:19,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:19,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692539935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692539935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692539935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692539935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T17:41:20,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T17:41:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/24276baec9914b558f88d43846cf6049 2024-11-15T17:41:20,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/54f9f6fdcc7340ad876e7842a54b29a0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/54f9f6fdcc7340ad876e7842a54b29a0 2024-11-15T17:41:20,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/54f9f6fdcc7340ad876e7842a54b29a0, entries=200, sequenceid=131, filesize=38.7 K 2024-11-15T17:41:20,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/fba3115320804120a2e84c831db2299d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/fba3115320804120a2e84c831db2299d 2024-11-15T17:41:20,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/fba3115320804120a2e84c831db2299d, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T17:41:20,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/24276baec9914b558f88d43846cf6049 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/24276baec9914b558f88d43846cf6049 2024-11-15T17:41:20,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/24276baec9914b558f88d43846cf6049, entries=150, sequenceid=131, filesize=11.8 K 2024-11-15T17:41:20,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 057f1b8c8fc0614d025e1dac4cdaab16 in 917ms, sequenceid=131, compaction requested=true 2024-11-15T17:41:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:20,137 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:20,137 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:20,138 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,138 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,138 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/c3d62611c1fe490cb62e85846cbe914f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e4a2f169c614f79a9a606464dd2b716, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/54f9f6fdcc7340ad876e7842a54b29a0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=107.8 K 2024-11-15T17:41:20,138 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/8695c0cdd23546e6a388f2a90b398052, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/55cb991ee1044a5cad0b10e18c5e4d78, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/fba3115320804120a2e84c831db2299d] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=35.5 K 2024-11-15T17:41:20,138 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/c3d62611c1fe490cb62e85846cbe914f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e4a2f169c614f79a9a606464dd2b716, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/54f9f6fdcc7340ad876e7842a54b29a0] 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8695c0cdd23546e6a388f2a90b398052, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692477257 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d62611c1fe490cb62e85846cbe914f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692477257 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 55cb991ee1044a5cad0b10e18c5e4d78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692478430 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e4a2f169c614f79a9a606464dd2b716, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692478430 2024-11-15T17:41:20,138 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fba3115320804120a2e84c831db2299d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731692479101 2024-11-15T17:41:20,139 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f9f6fdcc7340ad876e7842a54b29a0, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731692479100 2024-11-15T17:41:20,143 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:20,150 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115dd36acd05210463cba88c5573790fd46_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:20,152 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:20,152 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115dd36acd05210463cba88c5573790fd46_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:20,152 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/6f88596a2ebc4ae28ad8336949516099 is 50, key is test_row_0/B:col10/1731692479109/Put/seqid=0 2024-11-15T17:41:20,152 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dd36acd05210463cba88c5573790fd46_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:20,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742204_1380 (size=4469) 2024-11-15T17:41:20,159 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#319 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:20,160 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/d378008922bb43fabb4c02dc5b25cd81 is 175, key is test_row_0/A:col10/1731692479109/Put/seqid=0 2024-11-15T17:41:20,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742205_1381 (size=12409) 2024-11-15T17:41:20,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742206_1382 (size=31363) 2024-11-15T17:41:20,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T17:41:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,233 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T17:41:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111563d71745ddba45849233d9446942b3f6_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692479281/Put/seqid=0 2024-11-15T17:41:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T17:41:20,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742207_1383 (size=12304) 2024-11-15T17:41:20,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,253 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111563d71745ddba45849233d9446942b3f6_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563d71745ddba45849233d9446942b3f6_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:20,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e5b6b8342cd4962b6db9ab47c053271, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:20,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e5b6b8342cd4962b6db9ab47c053271 is 175, key is test_row_0/A:col10/1731692479281/Put/seqid=0 2024-11-15T17:41:20,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742208_1384 (size=31105) 2024-11-15T17:41:20,274 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e5b6b8342cd4962b6db9ab47c053271 2024-11-15T17:41:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/61e2c7f492c04b71b7d27be9c09fbc21 is 50, key is test_row_0/B:col10/1731692479281/Put/seqid=0 2024-11-15T17:41:20,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742209_1385 (size=12151) 2024-11-15T17:41:20,291 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/61e2c7f492c04b71b7d27be9c09fbc21 2024-11-15T17:41:20,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/9275bddf873f477e9c8d9c29a4067d2a is 50, key is test_row_0/C:col10/1731692479281/Put/seqid=0 2024-11-15T17:41:20,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742210_1386 (size=12151) 2024-11-15T17:41:20,317 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/9275bddf873f477e9c8d9c29a4067d2a 2024-11-15T17:41:20,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/2e5b6b8342cd4962b6db9ab47c053271 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e5b6b8342cd4962b6db9ab47c053271 2024-11-15T17:41:20,327 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e5b6b8342cd4962b6db9ab47c053271, entries=150, sequenceid=155, filesize=30.4 K 2024-11-15T17:41:20,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/61e2c7f492c04b71b7d27be9c09fbc21 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/61e2c7f492c04b71b7d27be9c09fbc21 2024-11-15T17:41:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,334 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/61e2c7f492c04b71b7d27be9c09fbc21, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T17:41:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/9275bddf873f477e9c8d9c29a4067d2a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9275bddf873f477e9c8d9c29a4067d2a 2024-11-15T17:41:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,337 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9275bddf873f477e9c8d9c29a4067d2a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T17:41:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,338 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 057f1b8c8fc0614d025e1dac4cdaab16 in 105ms, sequenceid=155, compaction requested=true 2024-11-15T17:41:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:20,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T17:41:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T17:41:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T17:41:20,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 411 msec 2024-11-15T17:41:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 417 msec 2024-11-15T17:41:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T17:41:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,555 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T17:41:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T17:41:20,557 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:20,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:20,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:41:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,575 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/6f88596a2ebc4ae28ad8336949516099 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f88596a2ebc4ae28ad8336949516099 2024-11-15T17:41:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,584 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/d378008922bb43fabb4c02dc5b25cd81 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d378008922bb43fabb4c02dc5b25cd81 2024-11-15T17:41:20,584 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 6f88596a2ebc4ae28ad8336949516099(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:20,584 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:20,584 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692480137; duration=0sec 2024-11-15T17:41:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,585 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:20,585 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:20,585 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,586 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:20,586 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,586 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,586 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6745bb5d09cc4227a8ef81f5bb4689ee, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/24276baec9914b558f88d43846cf6049, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9275bddf873f477e9c8d9c29a4067d2a] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=35.6 K 2024-11-15T17:41:20,587 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6745bb5d09cc4227a8ef81f5bb4689ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731692478430 2024-11-15T17:41:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,587 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 24276baec9914b558f88d43846cf6049, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731692479101 2024-11-15T17:41:20,587 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 9275bddf873f477e9c8d9c29a4067d2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692479263 2024-11-15T17:41:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,590 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into d378008922bb43fabb4c02dc5b25cd81(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:20,590 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:20,590 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692480137; duration=0sec 2024-11-15T17:41:20,590 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:20,590 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ece0b95efea44c6eb2a08dd57f334b65_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692480565/Put/seqid=0 2024-11-15T17:41:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,606 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,607 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/2cc497367e9c4323ab59cca71b842815 is 50, key is test_row_0/C:col10/1731692479281/Put/seqid=0 2024-11-15T17:41:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742212_1388 (size=24758) 2024-11-15T17:41:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,643 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ece0b95efea44c6eb2a08dd57f334b65_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ece0b95efea44c6eb2a08dd57f334b65_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:20,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/e2e7c0f80b4b41648120becd92e6782c, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:20,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/e2e7c0f80b4b41648120becd92e6782c is 175, key is test_row_0/A:col10/1731692480565/Put/seqid=0 2024-11-15T17:41:20,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742211_1387 (size=74395) 2024-11-15T17:41:20,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/e2e7c0f80b4b41648120becd92e6782c 2024-11-15T17:41:20,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742213_1389 (size=12493) 2024-11-15T17:41:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T17:41:20,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f7a79d407a4f4ded8abf357fdd2fe9d0 is 50, key is test_row_0/B:col10/1731692480565/Put/seqid=0 2024-11-15T17:41:20,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692540687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692540686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742214_1390 (size=12151) 2024-11-15T17:41:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692540698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T17:41:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692540701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692540805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692540806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692540816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692540826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:20,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T17:41:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:20,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T17:41:21,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T17:41:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:21,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:21,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692541016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692541016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692541035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692541035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,068 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/2cc497367e9c4323ab59cca71b842815 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/2cc497367e9c4323ab59cca71b842815 2024-11-15T17:41:21,072 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into 2cc497367e9c4323ab59cca71b842815(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:21,072 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:21,072 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=13, startTime=1731692480137; duration=0sec 2024-11-15T17:41:21,073 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:21,073 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:21,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f7a79d407a4f4ded8abf357fdd2fe9d0 2024-11-15T17:41:21,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/e31f445dbc8b4a8aac9024386325dc4e is 50, key is test_row_0/C:col10/1731692480565/Put/seqid=0 2024-11-15T17:41:21,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742215_1391 (size=12151) 2024-11-15T17:41:21,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/e31f445dbc8b4a8aac9024386325dc4e 2024-11-15T17:41:21,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/e2e7c0f80b4b41648120becd92e6782c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/e2e7c0f80b4b41648120becd92e6782c 2024-11-15T17:41:21,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/e2e7c0f80b4b41648120becd92e6782c, entries=400, sequenceid=166, filesize=72.7 K 2024-11-15T17:41:21,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f7a79d407a4f4ded8abf357fdd2fe9d0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f7a79d407a4f4ded8abf357fdd2fe9d0 2024-11-15T17:41:21,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f7a79d407a4f4ded8abf357fdd2fe9d0, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T17:41:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/e31f445dbc8b4a8aac9024386325dc4e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/e31f445dbc8b4a8aac9024386325dc4e 2024-11-15T17:41:21,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/e31f445dbc8b4a8aac9024386325dc4e, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T17:41:21,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 057f1b8c8fc0614d025e1dac4cdaab16 in 586ms, sequenceid=166, compaction requested=true 2024-11-15T17:41:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:21,157 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:21,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:21,158 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:21,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:21,158 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:21,158 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:21,158 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:21,158 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:21,158 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:21,158 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d378008922bb43fabb4c02dc5b25cd81, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e5b6b8342cd4962b6db9ab47c053271, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/e2e7c0f80b4b41648120becd92e6782c] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=133.7 K 2024-11-15T17:41:21,158 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:21,158 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:21,158 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d378008922bb43fabb4c02dc5b25cd81, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e5b6b8342cd4962b6db9ab47c053271, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/e2e7c0f80b4b41648120becd92e6782c] 2024-11-15T17:41:21,159 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f88596a2ebc4ae28ad8336949516099, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/61e2c7f492c04b71b7d27be9c09fbc21, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f7a79d407a4f4ded8abf357fdd2fe9d0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=35.9 K 2024-11-15T17:41:21,159 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d378008922bb43fabb4c02dc5b25cd81, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731692479101 2024-11-15T17:41:21,159 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e5b6b8342cd4962b6db9ab47c053271, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692479263 2024-11-15T17:41:21,160 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2e7c0f80b4b41648120becd92e6782c, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692480537 2024-11-15T17:41:21,166 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f88596a2ebc4ae28ad8336949516099, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731692479101 2024-11-15T17:41:21,166 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e2c7f492c04b71b7d27be9c09fbc21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692479263 2024-11-15T17:41:21,167 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a79d407a4f4ded8abf357fdd2fe9d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692480559 2024-11-15T17:41:21,172 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:21,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T17:41:21,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:21,175 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:41:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T17:41:21,189 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115724afa01b8344b4aa6e052c19c877da8_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:21,205 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:21,205 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/916d26f11dd043578b09e7ff95005ceb is 50, key is test_row_0/B:col10/1731692480565/Put/seqid=0 2024-11-15T17:41:21,210 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115724afa01b8344b4aa6e052c19c877da8_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:21,210 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115724afa01b8344b4aa6e052c19c877da8_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:21,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515fcfafd22f54bf893e632c811837708_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692480663/Put/seqid=0 2024-11-15T17:41:21,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742216_1392 (size=12561) 2024-11-15T17:41:21,267 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/916d26f11dd043578b09e7ff95005ceb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/916d26f11dd043578b09e7ff95005ceb 2024-11-15T17:41:21,273 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 916d26f11dd043578b09e7ff95005ceb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:21,273 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:21,273 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692481157; duration=0sec 2024-11-15T17:41:21,273 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:21,273 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:21,274 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T17:41:21,274 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T17:41:21,274 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T17:41:21,274 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. because compaction request was cancelled 2024-11-15T17:41:21,274 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:21,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742217_1393 (size=4469) 2024-11-15T17:41:21,286 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#328 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:21,287 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5fdf67fbb1334aee8c6e54f2932325a5 is 175, key is test_row_0/A:col10/1731692480565/Put/seqid=0 2024-11-15T17:41:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742218_1394 (size=12304) 2024-11-15T17:41:21,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:21,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742219_1395 (size=31515) 2024-11-15T17:41:21,366 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5fdf67fbb1334aee8c6e54f2932325a5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5fdf67fbb1334aee8c6e54f2932325a5 2024-11-15T17:41:21,372 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into 5fdf67fbb1334aee8c6e54f2932325a5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:21,372 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:21,372 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692481157; duration=0sec 2024-11-15T17:41:21,372 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:21,372 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692541369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692541371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692541372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692541374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692541495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692541495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692541496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692541496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T17:41:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:21,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692541706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692541706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692541706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692541707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:21,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515fcfafd22f54bf893e632c811837708_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515fcfafd22f54bf893e632c811837708_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4c5d9a34b4644c1bd100d413d98ca56, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:21,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4c5d9a34b4644c1bd100d413d98ca56 is 175, key is test_row_0/A:col10/1731692480663/Put/seqid=0 2024-11-15T17:41:21,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742220_1396 (size=31105) 2024-11-15T17:41:21,734 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4c5d9a34b4644c1bd100d413d98ca56 2024-11-15T17:41:21,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/86db8aa72d4b4229ad65833a1c739d86 is 50, key is test_row_0/B:col10/1731692480663/Put/seqid=0 2024-11-15T17:41:21,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742221_1397 (size=12151) 2024-11-15T17:41:21,758 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/86db8aa72d4b4229ad65833a1c739d86 2024-11-15T17:41:21,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/3897d5b83abd4be2b4642c88180ab52a is 50, key is test_row_0/C:col10/1731692480663/Put/seqid=0 2024-11-15T17:41:21,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742222_1398 (size=12151) 2024-11-15T17:41:22,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692542025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692542025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692542025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692542025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,188 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/3897d5b83abd4be2b4642c88180ab52a 2024-11-15T17:41:22,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4c5d9a34b4644c1bd100d413d98ca56 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4c5d9a34b4644c1bd100d413d98ca56 2024-11-15T17:41:22,199 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4c5d9a34b4644c1bd100d413d98ca56, entries=150, sequenceid=194, filesize=30.4 K 2024-11-15T17:41:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/86db8aa72d4b4229ad65833a1c739d86 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/86db8aa72d4b4229ad65833a1c739d86 2024-11-15T17:41:22,223 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/86db8aa72d4b4229ad65833a1c739d86, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T17:41:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/3897d5b83abd4be2b4642c88180ab52a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3897d5b83abd4be2b4642c88180ab52a 2024-11-15T17:41:22,228 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3897d5b83abd4be2b4642c88180ab52a, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T17:41:22,229 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1054ms, sequenceid=194, compaction requested=true 2024-11-15T17:41:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T17:41:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T17:41:22,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T17:41:22,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6730 sec 2024-11-15T17:41:22,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.6770 sec 2024-11-15T17:41:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:22,552 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:22,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:41:22,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111568d6815461414d77a2edd0e9f031ad61_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692482551/Put/seqid=0 2024-11-15T17:41:22,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742223_1399 (size=14794) 2024-11-15T17:41:22,567 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:22,571 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111568d6815461414d77a2edd0e9f031ad61_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111568d6815461414d77a2edd0e9f031ad61_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:22,572 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/4c0408615c424c26ae66243d5b09e22e, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:22,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/4c0408615c424c26ae66243d5b09e22e is 175, key is test_row_0/A:col10/1731692482551/Put/seqid=0 2024-11-15T17:41:22,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742224_1400 (size=39749) 2024-11-15T17:41:22,581 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/4c0408615c424c26ae66243d5b09e22e 2024-11-15T17:41:22,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/e70e39e6ea7841c8a5df93fcc642a829 is 50, key is test_row_0/B:col10/1731692482551/Put/seqid=0 2024-11-15T17:41:22,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742225_1401 (size=12151) 2024-11-15T17:41:22,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/e70e39e6ea7841c8a5df93fcc642a829 2024-11-15T17:41:22,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/78e972f79c034d26933f43bcdd2c7b69 is 50, key is test_row_0/C:col10/1731692482551/Put/seqid=0 2024-11-15T17:41:22,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742226_1402 (size=12151) 2024-11-15T17:41:22,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/78e972f79c034d26933f43bcdd2c7b69 2024-11-15T17:41:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/4c0408615c424c26ae66243d5b09e22e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/4c0408615c424c26ae66243d5b09e22e 2024-11-15T17:41:22,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692542615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692542618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692542618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/4c0408615c424c26ae66243d5b09e22e, entries=200, sequenceid=208, filesize=38.8 K 2024-11-15T17:41:22,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/e70e39e6ea7841c8a5df93fcc642a829 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e70e39e6ea7841c8a5df93fcc642a829 2024-11-15T17:41:22,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e70e39e6ea7841c8a5df93fcc642a829, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T17:41:22,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/78e972f79c034d26933f43bcdd2c7b69 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/78e972f79c034d26933f43bcdd2c7b69 2024-11-15T17:41:22,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/78e972f79c034d26933f43bcdd2c7b69, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T17:41:22,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 057f1b8c8fc0614d025e1dac4cdaab16 in 93ms, sequenceid=208, compaction requested=true 2024-11-15T17:41:22,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:22,646 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:22,646 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:22,646 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:22,647 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:22,647 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:22,647 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:22,647 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:22,647 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:22,647 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/916d26f11dd043578b09e7ff95005ceb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/86db8aa72d4b4229ad65833a1c739d86, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e70e39e6ea7841c8a5df93fcc642a829] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=36.0 K 2024-11-15T17:41:22,647 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5fdf67fbb1334aee8c6e54f2932325a5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4c5d9a34b4644c1bd100d413d98ca56, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/4c0408615c424c26ae66243d5b09e22e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=100.0 K 2024-11-15T17:41:22,647 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:22,647 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5fdf67fbb1334aee8c6e54f2932325a5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4c5d9a34b4644c1bd100d413d98ca56, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/4c0408615c424c26ae66243d5b09e22e] 2024-11-15T17:41:22,647 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 916d26f11dd043578b09e7ff95005ceb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692480559 2024-11-15T17:41:22,647 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 86db8aa72d4b4229ad65833a1c739d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731692480663 2024-11-15T17:41:22,647 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fdf67fbb1334aee8c6e54f2932325a5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692480559 2024-11-15T17:41:22,648 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting e70e39e6ea7841c8a5df93fcc642a829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731692481346 2024-11-15T17:41:22,648 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c5d9a34b4644c1bd100d413d98ca56, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731692480663 2024-11-15T17:41:22,648 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0408615c424c26ae66243d5b09e22e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731692481346 2024-11-15T17:41:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:22,648 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:22,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:41:22,674 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:22,674 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/13ea0a8e12724e6286fb39f5c516692a is 50, key is test_row_0/B:col10/1731692482551/Put/seqid=0 2024-11-15T17:41:22,681 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:22,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152b70b737bebe490c8f8e536c2d18b083_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692482615/Put/seqid=0 2024-11-15T17:41:22,703 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111515469bbad5e845a5ac9d4c213a61766b_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:22,705 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111515469bbad5e845a5ac9d4c213a61766b_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:22,705 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515469bbad5e845a5ac9d4c213a61766b_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T17:41:22,706 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:22,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742227_1403 (size=12663) 2024-11-15T17:41:22,707 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-15T17:41:22,709 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:22,710 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:22,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T17:41:22,714 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/13ea0a8e12724e6286fb39f5c516692a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/13ea0a8e12724e6286fb39f5c516692a 2024-11-15T17:41:22,719 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 13ea0a8e12724e6286fb39f5c516692a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:22,719 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:22,719 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692482646; duration=0sec 2024-11-15T17:41:22,719 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:22,719 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:22,719 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:22,721 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:22,721 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:22,721 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:22,721 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/2cc497367e9c4323ab59cca71b842815, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/e31f445dbc8b4a8aac9024386325dc4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3897d5b83abd4be2b4642c88180ab52a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/78e972f79c034d26933f43bcdd2c7b69] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=47.8 K 2024-11-15T17:41:22,723 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc497367e9c4323ab59cca71b842815, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692479263 2024-11-15T17:41:22,723 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting e31f445dbc8b4a8aac9024386325dc4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692480559 2024-11-15T17:41:22,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742228_1404 (size=14794) 2024-11-15T17:41:22,725 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 3897d5b83abd4be2b4642c88180ab52a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731692480663 2024-11-15T17:41:22,727 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e972f79c034d26933f43bcdd2c7b69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731692481346 2024-11-15T17:41:22,741 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:22,742 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/987abe95706b456697a3e9703e3ffbdc is 50, key is test_row_0/C:col10/1731692482551/Put/seqid=0 2024-11-15T17:41:22,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742229_1405 (size=4469) 2024-11-15T17:41:22,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692542736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692542743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742230_1406 (size=12629) 2024-11-15T17:41:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692542747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692542747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,761 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/987abe95706b456697a3e9703e3ffbdc as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/987abe95706b456697a3e9703e3ffbdc 2024-11-15T17:41:22,772 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into 987abe95706b456697a3e9703e3ffbdc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:22,772 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:22,772 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=12, startTime=1731692482646; duration=0sec 2024-11-15T17:41:22,773 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:22,773 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T17:41:22,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692542856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:22,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:22,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692542865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692542865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692542956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T17:41:23,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692543076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692543085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692543085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:23,131 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152b70b737bebe490c8f8e536c2d18b083_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152b70b737bebe490c8f8e536c2d18b083_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:23,132 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7890631770dc47d09837f8b215d18004, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:23,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7890631770dc47d09837f8b215d18004 is 175, key is test_row_0/A:col10/1731692482615/Put/seqid=0 2024-11-15T17:41:23,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742231_1407 (size=39749) 2024-11-15T17:41:23,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7890631770dc47d09837f8b215d18004 2024-11-15T17:41:23,145 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#337 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:23,145 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/82125a49d4c5446ba2e9bbfe1bbcc4b3 is 175, key is test_row_0/A:col10/1731692482551/Put/seqid=0 2024-11-15T17:41:23,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/b7f583e68db34504862ba4b5b9b40060 is 50, key is test_row_0/B:col10/1731692482615/Put/seqid=0 2024-11-15T17:41:23,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742232_1408 (size=31617) 2024-11-15T17:41:23,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,185 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/82125a49d4c5446ba2e9bbfe1bbcc4b3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/82125a49d4c5446ba2e9bbfe1bbcc4b3 2024-11-15T17:41:23,193 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into 82125a49d4c5446ba2e9bbfe1bbcc4b3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:23,193 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:23,193 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692482645; duration=0sec 2024-11-15T17:41:23,193 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:23,193 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:23,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742233_1409 (size=12151) 2024-11-15T17:41:23,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/b7f583e68db34504862ba4b5b9b40060 2024-11-15T17:41:23,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/d79de8df6bbe48259b05d40fbbb08c90 is 50, key is test_row_0/C:col10/1731692482615/Put/seqid=0 2024-11-15T17:41:23,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742234_1410 (size=12151) 2024-11-15T17:41:23,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692543266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T17:41:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692543396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692543396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692543396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:23,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/d79de8df6bbe48259b05d40fbbb08c90 2024-11-15T17:41:23,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/7890631770dc47d09837f8b215d18004 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7890631770dc47d09837f8b215d18004 2024-11-15T17:41:23,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7890631770dc47d09837f8b215d18004, entries=200, sequenceid=233, filesize=38.8 K 2024-11-15T17:41:23,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/b7f583e68db34504862ba4b5b9b40060 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b7f583e68db34504862ba4b5b9b40060 2024-11-15T17:41:23,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b7f583e68db34504862ba4b5b9b40060, entries=150, sequenceid=233, filesize=11.9 K 2024-11-15T17:41:23,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/d79de8df6bbe48259b05d40fbbb08c90 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/d79de8df6bbe48259b05d40fbbb08c90 2024-11-15T17:41:23,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/d79de8df6bbe48259b05d40fbbb08c90, entries=150, sequenceid=233, filesize=11.9 K 2024-11-15T17:41:23,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1010ms, sequenceid=233, compaction requested=false 2024-11-15T17:41:23,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:23,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:23,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:41:23,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151988433b5e45479ab2825acf64358fde_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692483779/Put/seqid=0 2024-11-15T17:41:23,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742235_1411 (size=14794) 2024-11-15T17:41:23,789 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:23,792 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151988433b5e45479ab2825acf64358fde_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151988433b5e45479ab2825acf64358fde_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:23,793 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/ca777da1da9e4c18ade400f6d7d8ee70, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:23,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/ca777da1da9e4c18ade400f6d7d8ee70 is 175, key is test_row_0/A:col10/1731692483779/Put/seqid=0 2024-11-15T17:41:23,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742236_1412 (size=39749) 2024-11-15T17:41:23,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:23,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:23,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T17:41:23,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692543923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692543928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692543930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692543937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:23,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:23,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:24,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692544045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692544045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692544046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692544046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:24,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:24,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:24,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:24,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/ca777da1da9e4c18ade400f6d7d8ee70 2024-11-15T17:41:24,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/7573a1ba84a048da8796e59c6c4b9799 is 50, key is test_row_0/B:col10/1731692483779/Put/seqid=0 2024-11-15T17:41:24,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742237_1413 (size=12151) 2024-11-15T17:41:24,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/7573a1ba84a048da8796e59c6c4b9799 2024-11-15T17:41:24,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/5c263516334240cabcdf84d90617d6ac is 50, key is test_row_0/C:col10/1731692483779/Put/seqid=0 2024-11-15T17:41:24,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:24,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742238_1414 (size=12151) 2024-11-15T17:41:24,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/5c263516334240cabcdf84d90617d6ac 2024-11-15T17:41:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/ca777da1da9e4c18ade400f6d7d8ee70 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ca777da1da9e4c18ade400f6d7d8ee70 2024-11-15T17:41:24,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ca777da1da9e4c18ade400f6d7d8ee70, entries=200, sequenceid=248, filesize=38.8 K 2024-11-15T17:41:24,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/7573a1ba84a048da8796e59c6c4b9799 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/7573a1ba84a048da8796e59c6c4b9799 2024-11-15T17:41:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692544257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692544257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/7573a1ba84a048da8796e59c6c4b9799, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T17:41:24,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/5c263516334240cabcdf84d90617d6ac as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5c263516334240cabcdf84d90617d6ac 2024-11-15T17:41:24,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5c263516334240cabcdf84d90617d6ac, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T17:41:24,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 057f1b8c8fc0614d025e1dac4cdaab16 in 510ms, sequenceid=248, compaction requested=true 2024-11-15T17:41:24,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:24,290 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:24,291 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:24,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:24,292 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:24,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:24,292 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:24,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:24,292 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,292 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,292 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/13ea0a8e12724e6286fb39f5c516692a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b7f583e68db34504862ba4b5b9b40060, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/7573a1ba84a048da8796e59c6c4b9799] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=36.1 K 2024-11-15T17:41:24,292 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/82125a49d4c5446ba2e9bbfe1bbcc4b3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7890631770dc47d09837f8b215d18004, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ca777da1da9e4c18ade400f6d7d8ee70] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=108.5 K 2024-11-15T17:41:24,292 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,292 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/82125a49d4c5446ba2e9bbfe1bbcc4b3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7890631770dc47d09837f8b215d18004, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ca777da1da9e4c18ade400f6d7d8ee70] 2024-11-15T17:41:24,295 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82125a49d4c5446ba2e9bbfe1bbcc4b3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731692481346 2024-11-15T17:41:24,295 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ea0a8e12724e6286fb39f5c516692a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731692481346 2024-11-15T17:41:24,295 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b7f583e68db34504862ba4b5b9b40060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731692482604 2024-11-15T17:41:24,295 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7890631770dc47d09837f8b215d18004, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731692482604 2024-11-15T17:41:24,295 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7573a1ba84a048da8796e59c6c4b9799, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731692482741 2024-11-15T17:41:24,295 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca777da1da9e4c18ade400f6d7d8ee70, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731692482741 2024-11-15T17:41:24,311 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:24,321 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a3e75bdbc57e4c56b29da3b99fbfea09_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:24,321 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:24,321 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/125fd34232b848d7b373b329d47a21e7 is 50, key is test_row_0/B:col10/1731692483779/Put/seqid=0 2024-11-15T17:41:24,322 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a3e75bdbc57e4c56b29da3b99fbfea09_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:24,322 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a3e75bdbc57e4c56b29da3b99fbfea09_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:24,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742239_1415 (size=12765) 2024-11-15T17:41:24,333 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/125fd34232b848d7b373b329d47a21e7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/125fd34232b848d7b373b329d47a21e7 2024-11-15T17:41:24,339 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 125fd34232b848d7b373b329d47a21e7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:24,339 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:24,339 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692484291; duration=0sec 2024-11-15T17:41:24,339 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:24,339 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:24,339 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:24,340 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:24,341 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:24,341 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,341 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/987abe95706b456697a3e9703e3ffbdc, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/d79de8df6bbe48259b05d40fbbb08c90, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5c263516334240cabcdf84d90617d6ac] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=36.1 K 2024-11-15T17:41:24,341 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 987abe95706b456697a3e9703e3ffbdc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731692481346 2024-11-15T17:41:24,341 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d79de8df6bbe48259b05d40fbbb08c90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731692482604 2024-11-15T17:41:24,342 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c263516334240cabcdf84d90617d6ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731692482741 2024-11-15T17:41:24,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742240_1416 (size=4469) 2024-11-15T17:41:24,346 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#345 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:24,347 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/782231a9b2004045953cdbe817e4d2d8 is 175, key is test_row_0/A:col10/1731692483779/Put/seqid=0 2024-11-15T17:41:24,361 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:24,361 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/8efc6c91d7714a40a1547cd8b8b144bd is 50, key is test_row_0/C:col10/1731692483779/Put/seqid=0 2024-11-15T17:41:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742241_1417 (size=31719) 2024-11-15T17:41:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742242_1418 (size=12731) 2024-11-15T17:41:24,396 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/782231a9b2004045953cdbe817e4d2d8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/782231a9b2004045953cdbe817e4d2d8 2024-11-15T17:41:24,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T17:41:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:24,412 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:41:24,412 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into 782231a9b2004045953cdbe817e4d2d8(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:24,412 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:24,412 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692484290; duration=0sec 2024-11-15T17:41:24,412 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:24,412 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115198b41fca3f842ac81f8ed850e4de781_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692483935/Put/seqid=0 2024-11-15T17:41:24,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742243_1419 (size=12454) 2024-11-15T17:41:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:24,457 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115198b41fca3f842ac81f8ed850e4de781_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115198b41fca3f842ac81f8ed850e4de781_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:24,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/d448baa1f6a44c2293a23b0722f0ba8a, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:24,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/d448baa1f6a44c2293a23b0722f0ba8a is 175, key is test_row_0/A:col10/1731692483935/Put/seqid=0 2024-11-15T17:41:24,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742244_1420 (size=31255) 2024-11-15T17:41:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:24,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:24,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692544591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692544592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692544592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692544595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692544705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692544705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692544705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692544705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,787 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/8efc6c91d7714a40a1547cd8b8b144bd as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/8efc6c91d7714a40a1547cd8b8b144bd 2024-11-15T17:41:24,791 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into 8efc6c91d7714a40a1547cd8b8b144bd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:24,791 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:24,791 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=13, startTime=1731692484291; duration=0sec 2024-11-15T17:41:24,791 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:24,791 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T17:41:24,877 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/d448baa1f6a44c2293a23b0722f0ba8a 2024-11-15T17:41:24,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/11476806fb8e47ba916e495b0fc40791 is 50, key is test_row_0/B:col10/1731692483935/Put/seqid=0 2024-11-15T17:41:24,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742245_1421 (size=12301) 2024-11-15T17:41:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692544915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692544915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692544916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692544916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692545226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692545226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692545226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692545235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,289 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/11476806fb8e47ba916e495b0fc40791 2024-11-15T17:41:25,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/a90f44b8aba1453ab8a2c03790332dc6 is 50, key is test_row_0/C:col10/1731692483935/Put/seqid=0 2024-11-15T17:41:25,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742246_1422 (size=12301) 2024-11-15T17:41:25,711 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/a90f44b8aba1453ab8a2c03790332dc6 2024-11-15T17:41:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/d448baa1f6a44c2293a23b0722f0ba8a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d448baa1f6a44c2293a23b0722f0ba8a 2024-11-15T17:41:25,724 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d448baa1f6a44c2293a23b0722f0ba8a, entries=150, sequenceid=274, filesize=30.5 K 2024-11-15T17:41:25,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/11476806fb8e47ba916e495b0fc40791 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/11476806fb8e47ba916e495b0fc40791 2024-11-15T17:41:25,737 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/11476806fb8e47ba916e495b0fc40791, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T17:41:25,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/a90f44b8aba1453ab8a2c03790332dc6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a90f44b8aba1453ab8a2c03790332dc6 2024-11-15T17:41:25,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692545739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:25,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692545741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,743 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a90f44b8aba1453ab8a2c03790332dc6, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T17:41:25,744 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1332ms, sequenceid=274, compaction requested=false 2024-11-15T17:41:25,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:25,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:25,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-15T17:41:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-15T17:41:25,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T17:41:25,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0360 sec 2024-11-15T17:41:25,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.0410 sec 2024-11-15T17:41:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:25,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:25,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:41:25,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111591e981160d244c1ca73998819dd63229_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692484586/Put/seqid=0 2024-11-15T17:41:25,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742247_1423 (size=12454) 2024-11-15T17:41:25,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:25,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111591e981160d244c1ca73998819dd63229_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111591e981160d244c1ca73998819dd63229_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:25,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/6ea457e90ebe40039125dd6df77d3a7c, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:25,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/6ea457e90ebe40039125dd6df77d3a7c is 175, key is test_row_0/A:col10/1731692484586/Put/seqid=0 2024-11-15T17:41:25,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742248_1424 (size=31255) 2024-11-15T17:41:25,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/6ea457e90ebe40039125dd6df77d3a7c 2024-11-15T17:41:25,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/e87bb71ed0984ea5a07985245f301a99 is 50, key is test_row_0/B:col10/1731692484586/Put/seqid=0 2024-11-15T17:41:25,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742249_1425 (size=12301) 2024-11-15T17:41:25,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/e87bb71ed0984ea5a07985245f301a99 2024-11-15T17:41:25,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/0b53a3db09c14388b8396b0a83553c2e is 50, key is test_row_0/C:col10/1731692484586/Put/seqid=0 2024-11-15T17:41:25,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742250_1426 (size=12301) 2024-11-15T17:41:25,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/0b53a3db09c14388b8396b0a83553c2e 2024-11-15T17:41:25,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/6ea457e90ebe40039125dd6df77d3a7c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/6ea457e90ebe40039125dd6df77d3a7c 2024-11-15T17:41:25,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/6ea457e90ebe40039125dd6df77d3a7c, entries=150, sequenceid=288, filesize=30.5 K 2024-11-15T17:41:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/e87bb71ed0984ea5a07985245f301a99 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e87bb71ed0984ea5a07985245f301a99 2024-11-15T17:41:25,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e87bb71ed0984ea5a07985245f301a99, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T17:41:25,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/0b53a3db09c14388b8396b0a83553c2e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/0b53a3db09c14388b8396b0a83553c2e 2024-11-15T17:41:25,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/0b53a3db09c14388b8396b0a83553c2e, entries=150, sequenceid=288, filesize=12.0 K 2024-11-15T17:41:25,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 057f1b8c8fc0614d025e1dac4cdaab16 in 126ms, sequenceid=288, compaction requested=true 2024-11-15T17:41:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:25,880 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:25,880 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:25,881 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:25,881 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:25,881 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:25,881 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/782231a9b2004045953cdbe817e4d2d8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d448baa1f6a44c2293a23b0722f0ba8a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/6ea457e90ebe40039125dd6df77d3a7c] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=92.0 K 2024-11-15T17:41:25,881 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:25,881 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/782231a9b2004045953cdbe817e4d2d8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d448baa1f6a44c2293a23b0722f0ba8a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/6ea457e90ebe40039125dd6df77d3a7c] 2024-11-15T17:41:25,882 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:25,882 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:25,882 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782231a9b2004045953cdbe817e4d2d8, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731692482741 2024-11-15T17:41:25,882 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:25,882 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/125fd34232b848d7b373b329d47a21e7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/11476806fb8e47ba916e495b0fc40791, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e87bb71ed0984ea5a07985245f301a99] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=36.5 K 2024-11-15T17:41:25,882 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d448baa1f6a44c2293a23b0722f0ba8a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731692483921 2024-11-15T17:41:25,882 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 125fd34232b848d7b373b329d47a21e7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731692482741 2024-11-15T17:41:25,884 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 11476806fb8e47ba916e495b0fc40791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731692483921 2024-11-15T17:41:25,884 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea457e90ebe40039125dd6df77d3a7c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731692484586 2024-11-15T17:41:25,885 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting e87bb71ed0984ea5a07985245f301a99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731692484586 2024-11-15T17:41:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:25,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:25,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:41:25,907 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:25,907 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/96ef35dad1cd47dd85cbfa1225df84a2 is 50, key is test_row_0/B:col10/1731692484586/Put/seqid=0 2024-11-15T17:41:25,915 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:25,955 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411154f15c0ce08eb403088e9ac79b260f69b_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:25,957 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411154f15c0ce08eb403088e9ac79b260f69b_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:25,957 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154f15c0ce08eb403088e9ac79b260f69b_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:25,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e753b002b4cf4ad0859e9d531d0202f7_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692485864/Put/seqid=0 2024-11-15T17:41:25,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692545967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:25,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:25,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742251_1427 (size=13017) 2024-11-15T17:41:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692545987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742252_1428 (size=4469) 2024-11-15T17:41:26,050 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#355 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:26,051 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/ba80ab9aebab4d528a63daff32a2e2d6 is 175, key is test_row_0/A:col10/1731692484586/Put/seqid=0 2024-11-15T17:41:26,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742253_1429 (size=12454) 2024-11-15T17:41:26,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742254_1430 (size=31971) 2024-11-15T17:41:26,107 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/ba80ab9aebab4d528a63daff32a2e2d6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ba80ab9aebab4d528a63daff32a2e2d6 2024-11-15T17:41:26,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692546095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,114 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into ba80ab9aebab4d528a63daff32a2e2d6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:26,114 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:26,114 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692485880; duration=0sec 2024-11-15T17:41:26,115 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:26,115 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:26,115 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:26,116 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:26,117 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:26,117 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:26,117 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/8efc6c91d7714a40a1547cd8b8b144bd, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a90f44b8aba1453ab8a2c03790332dc6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/0b53a3db09c14388b8396b0a83553c2e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=36.5 K 2024-11-15T17:41:26,117 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efc6c91d7714a40a1547cd8b8b144bd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731692482741 2024-11-15T17:41:26,118 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting a90f44b8aba1453ab8a2c03790332dc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731692483921 2024-11-15T17:41:26,119 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b53a3db09c14388b8396b0a83553c2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731692484586 2024-11-15T17:41:26,132 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:26,132 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/6c06f2ffc54042e4b39cefd2c2f7a941 is 50, key is test_row_0/C:col10/1731692484586/Put/seqid=0 2024-11-15T17:41:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692546116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742255_1431 (size=12983) 2024-11-15T17:41:26,170 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/6c06f2ffc54042e4b39cefd2c2f7a941 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6c06f2ffc54042e4b39cefd2c2f7a941 2024-11-15T17:41:26,174 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into 6c06f2ffc54042e4b39cefd2c2f7a941(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:26,174 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:26,175 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=13, startTime=1731692485880; duration=0sec 2024-11-15T17:41:26,175 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:26,175 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:26,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692546315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692546346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,396 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/96ef35dad1cd47dd85cbfa1225df84a2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/96ef35dad1cd47dd85cbfa1225df84a2 2024-11-15T17:41:26,400 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 96ef35dad1cd47dd85cbfa1225df84a2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:26,400 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:26,400 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692485880; duration=0sec 2024-11-15T17:41:26,400 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:26,400 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:26,468 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:26,471 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e753b002b4cf4ad0859e9d531d0202f7_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e753b002b4cf4ad0859e9d531d0202f7_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:26,472 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/a3a9648c9f6b48ac9408c9fb7b602a32, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:26,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/a3a9648c9f6b48ac9408c9fb7b602a32 is 175, key is test_row_0/A:col10/1731692485864/Put/seqid=0 2024-11-15T17:41:26,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742256_1432 (size=31255) 2024-11-15T17:41:26,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692546626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692546656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692546755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692546755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:26,760 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:26,760 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T17:41:26,865 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:26,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-15T17:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T17:41:26,867 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:26,868 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:26,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:26,877 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/a3a9648c9f6b48ac9408c9fb7b602a32 2024-11-15T17:41:26,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/87cc50598e364942b430d732b1ed8745 is 50, key is test_row_0/B:col10/1731692485864/Put/seqid=0 2024-11-15T17:41:26,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742257_1433 (size=12301) 2024-11-15T17:41:26,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/87cc50598e364942b430d732b1ed8745 2024-11-15T17:41:26,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/6fa6de1f51fc4fa2ba7e0a729ea78280 is 50, key is test_row_0/C:col10/1731692485864/Put/seqid=0 2024-11-15T17:41:26,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742258_1434 (size=12301) 2024-11-15T17:41:26,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/6fa6de1f51fc4fa2ba7e0a729ea78280 2024-11-15T17:41:26,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/a3a9648c9f6b48ac9408c9fb7b602a32 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/a3a9648c9f6b48ac9408c9fb7b602a32 2024-11-15T17:41:26,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/a3a9648c9f6b48ac9408c9fb7b602a32, entries=150, sequenceid=312, filesize=30.5 K 2024-11-15T17:41:26,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/87cc50598e364942b430d732b1ed8745 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/87cc50598e364942b430d732b1ed8745 2024-11-15T17:41:26,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/87cc50598e364942b430d732b1ed8745, entries=150, sequenceid=312, filesize=12.0 K 2024-11-15T17:41:26,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/6fa6de1f51fc4fa2ba7e0a729ea78280 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6fa6de1f51fc4fa2ba7e0a729ea78280 2024-11-15T17:41:26,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6fa6de1f51fc4fa2ba7e0a729ea78280, entries=150, sequenceid=312, filesize=12.0 K 2024-11-15T17:41:26,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1054ms, sequenceid=312, compaction requested=false 2024-11-15T17:41:26,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T17:41:27,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T17:41:27,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:27,020 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:41:27,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111561d57e4ac68842e8b19d353a5d44582f_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692485962/Put/seqid=0 2024-11-15T17:41:27,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742259_1435 (size=12454) 2024-11-15T17:41:27,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T17:41:27,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692547265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692547266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:27,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692547376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:27,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692547385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:27,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:27,445 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111561d57e4ac68842e8b19d353a5d44582f_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111561d57e4ac68842e8b19d353a5d44582f_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/775e755030a84518a79d03b9f9e2eb24, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/775e755030a84518a79d03b9f9e2eb24 is 175, key is test_row_0/A:col10/1731692485962/Put/seqid=0 2024-11-15T17:41:27,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742260_1436 (size=31255) 2024-11-15T17:41:27,454 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/775e755030a84518a79d03b9f9e2eb24 2024-11-15T17:41:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/d49a76035b1e40bdb9b620a70c857cb3 is 50, key is test_row_0/B:col10/1731692485962/Put/seqid=0 2024-11-15T17:41:27,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742261_1437 (size=12301) 2024-11-15T17:41:27,468 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/d49a76035b1e40bdb9b620a70c857cb3 2024-11-15T17:41:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/bb11acf82fab411bb8505f59c7196213 is 50, key is test_row_0/C:col10/1731692485962/Put/seqid=0 2024-11-15T17:41:27,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742262_1438 (size=12301) 2024-11-15T17:41:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T17:41:27,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692547586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:27,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692547595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:27,877 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/bb11acf82fab411bb8505f59c7196213 2024-11-15T17:41:27,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/775e755030a84518a79d03b9f9e2eb24 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/775e755030a84518a79d03b9f9e2eb24 2024-11-15T17:41:27,885 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/775e755030a84518a79d03b9f9e2eb24, entries=150, sequenceid=327, filesize=30.5 K 2024-11-15T17:41:27,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/d49a76035b1e40bdb9b620a70c857cb3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/d49a76035b1e40bdb9b620a70c857cb3 2024-11-15T17:41:27,889 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/d49a76035b1e40bdb9b620a70c857cb3, entries=150, sequenceid=327, filesize=12.0 K 2024-11-15T17:41:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/bb11acf82fab411bb8505f59c7196213 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/bb11acf82fab411bb8505f59c7196213 2024-11-15T17:41:27,893 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/bb11acf82fab411bb8505f59c7196213, entries=150, sequenceid=327, filesize=12.0 K 2024-11-15T17:41:27,893 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 057f1b8c8fc0614d025e1dac4cdaab16 in 873ms, sequenceid=327, compaction requested=true 2024-11-15T17:41:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:27,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-15T17:41:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-15T17:41:27,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T17:41:27,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0270 sec 2024-11-15T17:41:27,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0310 sec 2024-11-15T17:41:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:27,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:41:27,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515c007b1e22f42ffa76d5a9a5bcaddb9_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692487254/Put/seqid=0 2024-11-15T17:41:27,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742263_1439 (size=14994) 2024-11-15T17:41:27,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692547935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:27,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692547935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:27,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T17:41:28,006 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:28,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-15T17:41:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T17:41:28,008 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:28,009 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:28,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:28,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692548046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692548046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T17:41:28,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T17:41:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692548256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692548256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:28,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T17:41:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515c007b1e22f42ffa76d5a9a5bcaddb9_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515c007b1e22f42ffa76d5a9a5bcaddb9_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5294ff206ba24714993d11fafa1c526e, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:28,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5294ff206ba24714993d11fafa1c526e is 175, key is test_row_0/A:col10/1731692487254/Put/seqid=0 2024-11-15T17:41:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T17:41:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742264_1440 (size=39949) 2024-11-15T17:41:28,349 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5294ff206ba24714993d11fafa1c526e 2024-11-15T17:41:28,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/85acbfb3f1154e66ad518d18917d23ba is 50, key is test_row_0/B:col10/1731692487254/Put/seqid=0 2024-11-15T17:41:28,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742265_1441 (size=12301) 2024-11-15T17:41:28,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/85acbfb3f1154e66ad518d18917d23ba 2024-11-15T17:41:28,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/9f39659de27a46268af6c46b93f52f87 is 50, key is test_row_0/C:col10/1731692487254/Put/seqid=0 2024-11-15T17:41:28,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742266_1442 (size=12301) 2024-11-15T17:41:28,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T17:41:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692548566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692548566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T17:41:28,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:28,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T17:41:28,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T17:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:28,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46746 deadline: 1731692548785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46686 deadline: 1731692548785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:28,795 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,795 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_0' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:28,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/9f39659de27a46268af6c46b93f52f87 2024-11-15T17:41:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5294ff206ba24714993d11fafa1c526e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5294ff206ba24714993d11fafa1c526e 2024-11-15T17:41:28,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5294ff206ba24714993d11fafa1c526e, entries=200, sequenceid=352, filesize=39.0 K 2024-11-15T17:41:28,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/85acbfb3f1154e66ad518d18917d23ba as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/85acbfb3f1154e66ad518d18917d23ba 2024-11-15T17:41:28,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/85acbfb3f1154e66ad518d18917d23ba, entries=150, sequenceid=352, filesize=12.0 K 2024-11-15T17:41:28,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/9f39659de27a46268af6c46b93f52f87 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9f39659de27a46268af6c46b93f52f87 2024-11-15T17:41:28,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9f39659de27a46268af6c46b93f52f87, entries=150, sequenceid=352, filesize=12.0 K 2024-11-15T17:41:28,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 057f1b8c8fc0614d025e1dac4cdaab16 in 927ms, sequenceid=352, compaction requested=true 2024-11-15T17:41:28,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:28,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:28,828 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:28,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:28,828 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:28,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:28,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:28,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:28,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:28,829 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:28,829 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:28,830 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,830 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ba80ab9aebab4d528a63daff32a2e2d6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/a3a9648c9f6b48ac9408c9fb7b602a32, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/775e755030a84518a79d03b9f9e2eb24, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5294ff206ba24714993d11fafa1c526e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=131.3 K 2024-11-15T17:41:28,830 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,830 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ba80ab9aebab4d528a63daff32a2e2d6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/a3a9648c9f6b48ac9408c9fb7b602a32, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/775e755030a84518a79d03b9f9e2eb24, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5294ff206ba24714993d11fafa1c526e] 2024-11-15T17:41:28,830 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:28,830 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:28,830 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,830 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba80ab9aebab4d528a63daff32a2e2d6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731692484586 2024-11-15T17:41:28,830 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/96ef35dad1cd47dd85cbfa1225df84a2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/87cc50598e364942b430d732b1ed8745, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/d49a76035b1e40bdb9b620a70c857cb3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/85acbfb3f1154e66ad518d18917d23ba] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=48.8 K 2024-11-15T17:41:28,831 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3a9648c9f6b48ac9408c9fb7b602a32, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731692485864 2024-11-15T17:41:28,831 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ef35dad1cd47dd85cbfa1225df84a2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731692484586 2024-11-15T17:41:28,831 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 87cc50598e364942b430d732b1ed8745, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731692485864 2024-11-15T17:41:28,831 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775e755030a84518a79d03b9f9e2eb24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731692485923 2024-11-15T17:41:28,831 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d49a76035b1e40bdb9b620a70c857cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731692485923 2024-11-15T17:41:28,831 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5294ff206ba24714993d11fafa1c526e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731692487213 2024-11-15T17:41:28,831 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 85acbfb3f1154e66ad518d18917d23ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731692487254 2024-11-15T17:41:28,837 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:28,845 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115192fc6ce55a947a5a74c67bb8d85efd5_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:28,845 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:28,846 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/6f1a22e2913043368fcffac075ae0238 is 50, key is test_row_0/B:col10/1731692487254/Put/seqid=0 2024-11-15T17:41:28,848 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115192fc6ce55a947a5a74c67bb8d85efd5_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:28,848 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115192fc6ce55a947a5a74c67bb8d85efd5_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:28,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742267_1443 (size=13153) 2024-11-15T17:41:28,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742268_1444 (size=4469) 2024-11-15T17:41:28,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T17:41:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:28,936 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:41:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158a8fa9a215534bad8d75dcd57bcb31c8_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692487934/Put/seqid=0 2024-11-15T17:41:28,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742269_1445 (size=12454) 2024-11-15T17:41:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:29,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. as already flushing 2024-11-15T17:41:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T17:41:29,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692549215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692549223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,271 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:29,272 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/27b5d929d6284571b3c11afc7831b5f8 is 175, key is test_row_0/A:col10/1731692487254/Put/seqid=0 2024-11-15T17:41:29,276 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/6f1a22e2913043368fcffac075ae0238 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f1a22e2913043368fcffac075ae0238 2024-11-15T17:41:29,280 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into 6f1a22e2913043368fcffac075ae0238(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:29,281 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:29,281 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=12, startTime=1731692488828; duration=0sec 2024-11-15T17:41:29,281 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:29,281 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:29,281 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:29,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742270_1446 (size=32107) 2024-11-15T17:41:29,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:29,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:29,286 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:29,286 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6c06f2ffc54042e4b39cefd2c2f7a941, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6fa6de1f51fc4fa2ba7e0a729ea78280, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/bb11acf82fab411bb8505f59c7196213, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9f39659de27a46268af6c46b93f52f87] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=48.7 K 2024-11-15T17:41:29,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c06f2ffc54042e4b39cefd2c2f7a941, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731692484586 2024-11-15T17:41:29,287 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa6de1f51fc4fa2ba7e0a729ea78280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731692485864 2024-11-15T17:41:29,288 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bb11acf82fab411bb8505f59c7196213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731692485923 2024-11-15T17:41:29,288 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f39659de27a46268af6c46b93f52f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731692487254 2024-11-15T17:41:29,291 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/27b5d929d6284571b3c11afc7831b5f8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/27b5d929d6284571b3c11afc7831b5f8 2024-11-15T17:41:29,298 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into 27b5d929d6284571b3c11afc7831b5f8(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:29,298 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:29,298 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=12, startTime=1731692488828; duration=0sec 2024-11-15T17:41:29,298 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:29,298 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:29,301 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:29,301 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/664a6887601f41f281bb62e5c61164a0 is 50, key is test_row_0/C:col10/1731692487254/Put/seqid=0 2024-11-15T17:41:29,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742271_1447 (size=13119) 2024-11-15T17:41:29,313 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/664a6887601f41f281bb62e5c61164a0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/664a6887601f41f281bb62e5c61164a0 2024-11-15T17:41:29,318 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into 664a6887601f41f281bb62e5c61164a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:29,319 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:29,319 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=12, startTime=1731692488829; duration=0sec 2024-11-15T17:41:29,319 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:29,319 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:29,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692549335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692549335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:29,364 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158a8fa9a215534bad8d75dcd57bcb31c8_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a8fa9a215534bad8d75dcd57bcb31c8_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/434093e0ee184ec9b7da52ed584b058f, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:29,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/434093e0ee184ec9b7da52ed584b058f is 175, key is test_row_0/A:col10/1731692487934/Put/seqid=0 2024-11-15T17:41:29,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742272_1448 (size=31255) 2024-11-15T17:41:29,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692549556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692549556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,772 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/434093e0ee184ec9b7da52ed584b058f 2024-11-15T17:41:29,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/1ef77781f0d94712a55fbe8603c6e0db is 50, key is test_row_0/B:col10/1731692487934/Put/seqid=0 2024-11-15T17:41:29,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742273_1449 (size=12301) 2024-11-15T17:41:29,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692549865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692549875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:29,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46756 deadline: 1731692549995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:29,999 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in 057f1b8c8fc0614d025e1dac4cdaab16 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18345 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T17:41:30,182 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/1ef77781f0d94712a55fbe8603c6e0db 2024-11-15T17:41:30,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/29bfeebd0bf94347bd45960a138cdc8e is 50, key is test_row_0/C:col10/1731692487934/Put/seqid=0 2024-11-15T17:41:30,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742274_1450 (size=12301) 2024-11-15T17:41:30,205 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/29bfeebd0bf94347bd45960a138cdc8e 2024-11-15T17:41:30,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/434093e0ee184ec9b7da52ed584b058f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/434093e0ee184ec9b7da52ed584b058f 2024-11-15T17:41:30,218 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/434093e0ee184ec9b7da52ed584b058f, entries=150, sequenceid=363, filesize=30.5 K 2024-11-15T17:41:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/1ef77781f0d94712a55fbe8603c6e0db as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1ef77781f0d94712a55fbe8603c6e0db 2024-11-15T17:41:30,223 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1ef77781f0d94712a55fbe8603c6e0db, entries=150, sequenceid=363, filesize=12.0 K 2024-11-15T17:41:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/29bfeebd0bf94347bd45960a138cdc8e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/29bfeebd0bf94347bd45960a138cdc8e 2024-11-15T17:41:30,229 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/29bfeebd0bf94347bd45960a138cdc8e, entries=150, sequenceid=363, filesize=12.0 K 2024-11-15T17:41:30,230 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1294ms, sequenceid=363, compaction requested=false 2024-11-15T17:41:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-15T17:41:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-15T17:41:30,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T17:41:30,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2230 sec 2024-11-15T17:41:30,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.2290 sec 2024-11-15T17:41:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:30,383 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:30,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T17:41:30,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151803ef8a127c42d49f1620f5a7748288_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692490382/Put/seqid=0 2024-11-15T17:41:30,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742275_1451 (size=14994) 2024-11-15T17:41:30,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692550407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:30,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151803ef8a127c42d49f1620f5a7748288_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151803ef8a127c42d49f1620f5a7748288_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:30,420 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4e1655d187d48448c52bbdd48356f48, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:30,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4e1655d187d48448c52bbdd48356f48 is 175, key is test_row_0/A:col10/1731692490382/Put/seqid=0 2024-11-15T17:41:30,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692550419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:30,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742276_1452 (size=39949) 2024-11-15T17:41:30,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692550526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692550536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692550735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:30,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692550745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:30,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:30,834 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4e1655d187d48448c52bbdd48356f48 2024-11-15T17:41:30,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f67b48bf1a364a7890e3cdd1dde22690 is 50, key is test_row_0/B:col10/1731692490382/Put/seqid=0 2024-11-15T17:41:30,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742277_1453 (size=12301) 2024-11-15T17:41:31,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46776 deadline: 1731692551045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:31,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46770 deadline: 1731692551055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:41:31,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f67b48bf1a364a7890e3cdd1dde22690 2024-11-15T17:41:31,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/1e481d63379643e097ad261e412b8bc8 is 50, key is test_row_0/C:col10/1731692490382/Put/seqid=0 2024-11-15T17:41:31,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742278_1454 (size=12301) 2024-11-15T17:41:31,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/1e481d63379643e097ad261e412b8bc8 2024-11-15T17:41:31,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/f4e1655d187d48448c52bbdd48356f48 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4e1655d187d48448c52bbdd48356f48 2024-11-15T17:41:31,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4e1655d187d48448c52bbdd48356f48, entries=200, sequenceid=392, filesize=39.0 K 2024-11-15T17:41:31,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/f67b48bf1a364a7890e3cdd1dde22690 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f67b48bf1a364a7890e3cdd1dde22690 2024-11-15T17:41:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f67b48bf1a364a7890e3cdd1dde22690, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T17:41:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/1e481d63379643e097ad261e412b8bc8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/1e481d63379643e097ad261e412b8bc8 2024-11-15T17:41:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/1e481d63379643e097ad261e412b8bc8, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T17:41:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 057f1b8c8fc0614d025e1dac4cdaab16 in 900ms, sequenceid=392, compaction requested=true 2024-11-15T17:41:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:31,284 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:31,285 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057f1b8c8fc0614d025e1dac4cdaab16:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,285 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,286 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/A is initiating minor compaction (all files) 2024-11-15T17:41:31,286 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/A in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:31,286 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/27b5d929d6284571b3c11afc7831b5f8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/434093e0ee184ec9b7da52ed584b058f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4e1655d187d48448c52bbdd48356f48] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=100.9 K 2024-11-15T17:41:31,286 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:31,286 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/27b5d929d6284571b3c11afc7831b5f8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/434093e0ee184ec9b7da52ed584b058f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4e1655d187d48448c52bbdd48356f48] 2024-11-15T17:41:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,286 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b5d929d6284571b3c11afc7831b5f8, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731692487254 2024-11-15T17:41:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:31,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/B is initiating minor compaction (all files) 2024-11-15T17:41:31,286 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/B in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,286 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f1a22e2913043368fcffac075ae0238, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1ef77781f0d94712a55fbe8603c6e0db, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f67b48bf1a364a7890e3cdd1dde22690] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=36.9 K 2024-11-15T17:41:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,287 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 434093e0ee184ec9b7da52ed584b058f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731692487926 2024-11-15T17:41:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,287 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4e1655d187d48448c52bbdd48356f48, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731692489196 2024-11-15T17:41:31,287 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1a22e2913043368fcffac075ae0238, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731692487254 2024-11-15T17:41:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,288 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef77781f0d94712a55fbe8603c6e0db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731692487926 2024-11-15T17:41:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,288 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f67b48bf1a364a7890e3cdd1dde22690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731692489196 2024-11-15T17:41:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,293 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,295 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,295 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/b50345ed693d480f86c09d2e70aece1f is 50, key is test_row_0/B:col10/1731692490382/Put/seqid=0 2024-11-15T17:41:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,296 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411155f1d7ebf704e4093948621370d236e51_057f1b8c8fc0614d025e1dac4cdaab16 store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,298 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411155f1d7ebf704e4093948621370d236e51_057f1b8c8fc0614d025e1dac4cdaab16, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:31,298 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155f1d7ebf704e4093948621370d236e51_057f1b8c8fc0614d025e1dac4cdaab16 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742279_1455 (size=4469) 2024-11-15T17:41:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,302 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#A#compaction#375 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,303 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1067be3957a643ff86c8511580c71fcb is 175, key is test_row_0/A:col10/1731692490382/Put/seqid=0 2024-11-15T17:41:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742280_1456 (size=13255) 2024-11-15T17:41:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,312 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/b50345ed693d480f86c09d2e70aece1f as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b50345ed693d480f86c09d2e70aece1f 2024-11-15T17:41:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742281_1457 (size=32209) 2024-11-15T17:41:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,324 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/B of 057f1b8c8fc0614d025e1dac4cdaab16 into b50345ed693d480f86c09d2e70aece1f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:31,324 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:31,324 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/B, priority=13, startTime=1731692491285; duration=0sec 2024-11-15T17:41:31,324 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:31,324 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:B 2024-11-15T17:41:31,324 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,325 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:31,325 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 057f1b8c8fc0614d025e1dac4cdaab16/C is initiating minor compaction (all files) 2024-11-15T17:41:31,325 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057f1b8c8fc0614d025e1dac4cdaab16/C in TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:31,325 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/664a6887601f41f281bb62e5c61164a0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/29bfeebd0bf94347bd45960a138cdc8e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/1e481d63379643e097ad261e412b8bc8] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp, totalSize=36.8 K 2024-11-15T17:41:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,326 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 664a6887601f41f281bb62e5c61164a0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731692487254 2024-11-15T17:41:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,326 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 29bfeebd0bf94347bd45960a138cdc8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731692487926 2024-11-15T17:41:31,327 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e481d63379643e097ad261e412b8bc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731692489196 2024-11-15T17:41:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,340 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057f1b8c8fc0614d025e1dac4cdaab16#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,341 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/64e373216bec4d1e85082578a1795cc9 is 50, key is test_row_0/C:col10/1731692490382/Put/seqid=0 2024-11-15T17:41:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742282_1458 (size=13221) 2024-11-15T17:41:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,368 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/64e373216bec4d1e85082578a1795cc9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/64e373216bec4d1e85082578a1795cc9 2024-11-15T17:41:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,373 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/C of 057f1b8c8fc0614d025e1dac4cdaab16 into 64e373216bec4d1e85082578a1795cc9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:31,373 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:31,373 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/C, priority=13, startTime=1731692491285; duration=0sec 2024-11-15T17:41:31,373 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:31,373 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:C 2024-11-15T17:41:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:31,585 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:31,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:41:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c4ba038b0114f6bacac88d2b7c248e0_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_1/A:col10/1731692491584/Put/seqid=0 2024-11-15T17:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,605 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-15T17:41:31,605 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:31,605 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,605 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:31,606 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-15T17:41:31,606 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:31,606 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,606 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,623 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-15T17:41:31,623 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:31,623 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,623 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,627 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-15T17:41:31,627 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,627 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,627 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,627 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-15T17:41:31,627 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:31,627 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,627 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742283_1459 (size=12454) 2024-11-15T17:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,654 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-15T17:41:31,654 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:31,654 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,654 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:31,663 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-15T17:41:31,663 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:31,663 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,663 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:31,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:31,718 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/1067be3957a643ff86c8511580c71fcb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1067be3957a643ff86c8511580c71fcb 2024-11-15T17:41:31,723 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057f1b8c8fc0614d025e1dac4cdaab16/A of 057f1b8c8fc0614d025e1dac4cdaab16 into 1067be3957a643ff86c8511580c71fcb(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:31,723 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:31,723 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., storeName=057f1b8c8fc0614d025e1dac4cdaab16/A, priority=13, startTime=1731692491284; duration=0sec 2024-11-15T17:41:31,723 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:31,723 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057f1b8c8fc0614d025e1dac4cdaab16:A 2024-11-15T17:41:32,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:32,039 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c4ba038b0114f6bacac88d2b7c248e0_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c4ba038b0114f6bacac88d2b7c248e0_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:32,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/08f8481335a644cc84fc1e108d1560e0, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:32,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/08f8481335a644cc84fc1e108d1560e0 is 175, key is test_row_1/A:col10/1731692491584/Put/seqid=0 2024-11-15T17:41:32,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742284_1460 (size=31251) 2024-11-15T17:41:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T17:41:32,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:32,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/08f8481335a644cc84fc1e108d1560e0 2024-11-15T17:41:32,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/58a45f227a1e4fee8035befad0512f6a is 50, key is test_row_1/B:col10/1731692491584/Put/seqid=0 2024-11-15T17:41:32,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742285_1461 (size=9857) 2024-11-15T17:41:32,818 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-15T17:41:32,818 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:32,818 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:32,818 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:32,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:32,838 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-11-15T17:41:32,838 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:32,838 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:32,838 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:32,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:32,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/58a45f227a1e4fee8035befad0512f6a 2024-11-15T17:41:32,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/47e2c21734494862baa8c062cdcd48a9 is 50, key is test_row_1/C:col10/1731692491584/Put/seqid=0 2024-11-15T17:41:32,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742286_1462 (size=9857) 2024-11-15T17:41:33,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/47e2c21734494862baa8c062cdcd48a9 2024-11-15T17:41:33,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/08f8481335a644cc84fc1e108d1560e0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/08f8481335a644cc84fc1e108d1560e0 2024-11-15T17:41:33,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/08f8481335a644cc84fc1e108d1560e0, entries=150, sequenceid=405, filesize=30.5 K 2024-11-15T17:41:33,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/58a45f227a1e4fee8035befad0512f6a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/58a45f227a1e4fee8035befad0512f6a 2024-11-15T17:41:33,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/58a45f227a1e4fee8035befad0512f6a, entries=100, sequenceid=405, filesize=9.6 K 2024-11-15T17:41:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/47e2c21734494862baa8c062cdcd48a9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/47e2c21734494862baa8c062cdcd48a9 2024-11-15T17:41:33,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/47e2c21734494862baa8c062cdcd48a9, entries=100, sequenceid=405, filesize=9.6 K 2024-11-15T17:41:33,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=26.84 KB/27480 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1712ms, sequenceid=405, compaction requested=false 2024-11-15T17:41:33,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057f1b8c8fc0614d025e1dac4cdaab16: 2024-11-15T17:41:40,091 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-15T17:41:40,091 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:41:40,091 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:40,091 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:40,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:40,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:41:40,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-15T17:41:40,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-15T17:41:40,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-15T17:41:40,092 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1365 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4092 rows 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1373 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4111 rows 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1370 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4107 rows 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1371 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4110 rows 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1377 2024-11-15T17:41:40,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4128 rows 2024-11-15T17:41:40,093 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:41:40,093 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:41:40,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:40,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:40,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:41:40,095 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:41:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T17:41:40,100 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692500099"}]},"ts":"1731692500099"} 2024-11-15T17:41:40,101 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:41:40,101 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:41:40,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:41:40,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, UNASSIGN}] 2024-11-15T17:41:40,104 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, UNASSIGN 2024-11-15T17:41:40,105 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:40,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, UNASSIGN because future has completed 2024-11-15T17:41:40,107 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:41:40,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:41:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T17:41:40,260 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:40,260 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:41:40,260 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 057f1b8c8fc0614d025e1dac4cdaab16, disabling compactions & flushes 2024-11-15T17:41:40,260 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:40,260 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:40,260 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. after waiting 0 ms 2024-11-15T17:41:40,260 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:40,261 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 057f1b8c8fc0614d025e1dac4cdaab16 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T17:41:40,273 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c208e13ff6047f6b1db356d410f15dd_057f1b8c8fc0614d025e1dac4cdaab16 is 50, key is test_row_0/A:col10/1731692492836/Put/seqid=0 2024-11-15T17:41:40,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742287_1463 (size=12454) 2024-11-15T17:41:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T17:41:40,679 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:40,685 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c208e13ff6047f6b1db356d410f15dd_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c208e13ff6047f6b1db356d410f15dd_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:40,687 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5557683fc4144ec58610fcc0164830d1, store: [table=TestAcidGuarantees family=A region=057f1b8c8fc0614d025e1dac4cdaab16] 2024-11-15T17:41:40,688 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5557683fc4144ec58610fcc0164830d1 is 175, key is test_row_0/A:col10/1731692492836/Put/seqid=0 2024-11-15T17:41:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742288_1464 (size=31255) 2024-11-15T17:41:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T17:41:41,094 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5557683fc4144ec58610fcc0164830d1 2024-11-15T17:41:41,106 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/1883ece318934bd6b5b7e3ff6d215ae8 is 50, key is test_row_0/B:col10/1731692492836/Put/seqid=0 2024-11-15T17:41:41,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742289_1465 (size=12301) 2024-11-15T17:41:41,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:41:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T17:41:41,511 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/1883ece318934bd6b5b7e3ff6d215ae8 2024-11-15T17:41:41,523 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/5fbc35ef74254ef68887c701f06502f4 is 50, key is test_row_0/C:col10/1731692492836/Put/seqid=0 2024-11-15T17:41:41,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742290_1466 (size=12301) 2024-11-15T17:41:41,927 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/5fbc35ef74254ef68887c701f06502f4 2024-11-15T17:41:41,931 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/A/5557683fc4144ec58610fcc0164830d1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5557683fc4144ec58610fcc0164830d1 2024-11-15T17:41:41,934 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5557683fc4144ec58610fcc0164830d1, entries=150, sequenceid=414, filesize=30.5 K 2024-11-15T17:41:41,935 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/B/1883ece318934bd6b5b7e3ff6d215ae8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1883ece318934bd6b5b7e3ff6d215ae8 2024-11-15T17:41:41,939 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1883ece318934bd6b5b7e3ff6d215ae8, entries=150, sequenceid=414, filesize=12.0 K 2024-11-15T17:41:41,939 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/.tmp/C/5fbc35ef74254ef68887c701f06502f4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5fbc35ef74254ef68887c701f06502f4 2024-11-15T17:41:41,943 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5fbc35ef74254ef68887c701f06502f4, entries=150, sequenceid=414, filesize=12.0 K 2024-11-15T17:41:41,944 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1683ms, sequenceid=414, compaction requested=true 2024-11-15T17:41:41,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1521e9ffbfd547579733a8d6c8190cbe, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1e2d879051114a7da44969aee7204308, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7e3e63ce283f451d9c93e49ea1ef9928, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/50040307041b4cc69aadaceec595cb5f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/590ec6ef1ce74ce583575c98091b3232, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/fc9b93741b8f4ad78f4db70b52e93137, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/c3d62611c1fe490cb62e85846cbe914f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e4a2f169c614f79a9a606464dd2b716, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/54f9f6fdcc7340ad876e7842a54b29a0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d378008922bb43fabb4c02dc5b25cd81, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e5b6b8342cd4962b6db9ab47c053271, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/e2e7c0f80b4b41648120becd92e6782c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5fdf67fbb1334aee8c6e54f2932325a5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4c5d9a34b4644c1bd100d413d98ca56, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/4c0408615c424c26ae66243d5b09e22e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/82125a49d4c5446ba2e9bbfe1bbcc4b3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7890631770dc47d09837f8b215d18004, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ca777da1da9e4c18ade400f6d7d8ee70, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/782231a9b2004045953cdbe817e4d2d8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d448baa1f6a44c2293a23b0722f0ba8a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ba80ab9aebab4d528a63daff32a2e2d6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/6ea457e90ebe40039125dd6df77d3a7c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/a3a9648c9f6b48ac9408c9fb7b602a32, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/775e755030a84518a79d03b9f9e2eb24, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5294ff206ba24714993d11fafa1c526e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/27b5d929d6284571b3c11afc7831b5f8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/434093e0ee184ec9b7da52ed584b058f, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4e1655d187d48448c52bbdd48356f48] to archive 2024-11-15T17:41:41,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:41:41,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1521e9ffbfd547579733a8d6c8190cbe to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1521e9ffbfd547579733a8d6c8190cbe 2024-11-15T17:41:41,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1e2d879051114a7da44969aee7204308 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1e2d879051114a7da44969aee7204308 2024-11-15T17:41:41,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7e3e63ce283f451d9c93e49ea1ef9928 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7e3e63ce283f451d9c93e49ea1ef9928 2024-11-15T17:41:41,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/50040307041b4cc69aadaceec595cb5f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/50040307041b4cc69aadaceec595cb5f 2024-11-15T17:41:41,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/590ec6ef1ce74ce583575c98091b3232 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/590ec6ef1ce74ce583575c98091b3232 2024-11-15T17:41:41,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/fc9b93741b8f4ad78f4db70b52e93137 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/fc9b93741b8f4ad78f4db70b52e93137 2024-11-15T17:41:41,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/c3d62611c1fe490cb62e85846cbe914f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/c3d62611c1fe490cb62e85846cbe914f 2024-11-15T17:41:41,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e4a2f169c614f79a9a606464dd2b716 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e4a2f169c614f79a9a606464dd2b716 2024-11-15T17:41:41,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/54f9f6fdcc7340ad876e7842a54b29a0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/54f9f6fdcc7340ad876e7842a54b29a0 2024-11-15T17:41:41,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d378008922bb43fabb4c02dc5b25cd81 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d378008922bb43fabb4c02dc5b25cd81 2024-11-15T17:41:41,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e5b6b8342cd4962b6db9ab47c053271 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/2e5b6b8342cd4962b6db9ab47c053271 2024-11-15T17:41:41,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/e2e7c0f80b4b41648120becd92e6782c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/e2e7c0f80b4b41648120becd92e6782c 2024-11-15T17:41:41,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5fdf67fbb1334aee8c6e54f2932325a5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5fdf67fbb1334aee8c6e54f2932325a5 2024-11-15T17:41:41,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4c5d9a34b4644c1bd100d413d98ca56 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4c5d9a34b4644c1bd100d413d98ca56 2024-11-15T17:41:41,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/4c0408615c424c26ae66243d5b09e22e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/4c0408615c424c26ae66243d5b09e22e 2024-11-15T17:41:41,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/82125a49d4c5446ba2e9bbfe1bbcc4b3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/82125a49d4c5446ba2e9bbfe1bbcc4b3 2024-11-15T17:41:41,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7890631770dc47d09837f8b215d18004 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/7890631770dc47d09837f8b215d18004 2024-11-15T17:41:41,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ca777da1da9e4c18ade400f6d7d8ee70 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ca777da1da9e4c18ade400f6d7d8ee70 2024-11-15T17:41:41,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/782231a9b2004045953cdbe817e4d2d8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/782231a9b2004045953cdbe817e4d2d8 2024-11-15T17:41:41,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d448baa1f6a44c2293a23b0722f0ba8a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/d448baa1f6a44c2293a23b0722f0ba8a 2024-11-15T17:41:41,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ba80ab9aebab4d528a63daff32a2e2d6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/ba80ab9aebab4d528a63daff32a2e2d6 2024-11-15T17:41:41,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/6ea457e90ebe40039125dd6df77d3a7c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/6ea457e90ebe40039125dd6df77d3a7c 2024-11-15T17:41:41,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/a3a9648c9f6b48ac9408c9fb7b602a32 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/a3a9648c9f6b48ac9408c9fb7b602a32 2024-11-15T17:41:41,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/775e755030a84518a79d03b9f9e2eb24 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/775e755030a84518a79d03b9f9e2eb24 2024-11-15T17:41:41,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5294ff206ba24714993d11fafa1c526e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5294ff206ba24714993d11fafa1c526e 2024-11-15T17:41:41,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/27b5d929d6284571b3c11afc7831b5f8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/27b5d929d6284571b3c11afc7831b5f8 2024-11-15T17:41:41,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/434093e0ee184ec9b7da52ed584b058f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/434093e0ee184ec9b7da52ed584b058f 2024-11-15T17:41:41,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4e1655d187d48448c52bbdd48356f48 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/f4e1655d187d48448c52bbdd48356f48 2024-11-15T17:41:41,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/04c25b8026894357b2f1001270cc1ea8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/743020fe649a4407a26b22678b4e6588, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/36ae0aa2b41249fb83130f7e4d15d237, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/2b4c65f16045412db06caa9ba0c7672d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f6cf921b043140b3aa1c77d6269aa782, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/8695c0cdd23546e6a388f2a90b398052, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/ff6ba109620d4986962ded2b535dc70e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/55cb991ee1044a5cad0b10e18c5e4d78, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f88596a2ebc4ae28ad8336949516099, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/fba3115320804120a2e84c831db2299d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/61e2c7f492c04b71b7d27be9c09fbc21, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/916d26f11dd043578b09e7ff95005ceb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f7a79d407a4f4ded8abf357fdd2fe9d0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/86db8aa72d4b4229ad65833a1c739d86, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/13ea0a8e12724e6286fb39f5c516692a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e70e39e6ea7841c8a5df93fcc642a829, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b7f583e68db34504862ba4b5b9b40060, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/125fd34232b848d7b373b329d47a21e7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/7573a1ba84a048da8796e59c6c4b9799, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/11476806fb8e47ba916e495b0fc40791, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/96ef35dad1cd47dd85cbfa1225df84a2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e87bb71ed0984ea5a07985245f301a99, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/87cc50598e364942b430d732b1ed8745, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/d49a76035b1e40bdb9b620a70c857cb3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f1a22e2913043368fcffac075ae0238, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/85acbfb3f1154e66ad518d18917d23ba, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1ef77781f0d94712a55fbe8603c6e0db, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f67b48bf1a364a7890e3cdd1dde22690] to archive 2024-11-15T17:41:41,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:41:41,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/04c25b8026894357b2f1001270cc1ea8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/04c25b8026894357b2f1001270cc1ea8 2024-11-15T17:41:41,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/743020fe649a4407a26b22678b4e6588 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/743020fe649a4407a26b22678b4e6588 2024-11-15T17:41:41,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/36ae0aa2b41249fb83130f7e4d15d237 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/36ae0aa2b41249fb83130f7e4d15d237 2024-11-15T17:41:41,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/2b4c65f16045412db06caa9ba0c7672d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/2b4c65f16045412db06caa9ba0c7672d 2024-11-15T17:41:41,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f6cf921b043140b3aa1c77d6269aa782 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f6cf921b043140b3aa1c77d6269aa782 2024-11-15T17:41:41,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/8695c0cdd23546e6a388f2a90b398052 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/8695c0cdd23546e6a388f2a90b398052 2024-11-15T17:41:41,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/ff6ba109620d4986962ded2b535dc70e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/ff6ba109620d4986962ded2b535dc70e 2024-11-15T17:41:41,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/55cb991ee1044a5cad0b10e18c5e4d78 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/55cb991ee1044a5cad0b10e18c5e4d78 2024-11-15T17:41:41,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f88596a2ebc4ae28ad8336949516099 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f88596a2ebc4ae28ad8336949516099 2024-11-15T17:41:41,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/fba3115320804120a2e84c831db2299d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/fba3115320804120a2e84c831db2299d 2024-11-15T17:41:41,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/61e2c7f492c04b71b7d27be9c09fbc21 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/61e2c7f492c04b71b7d27be9c09fbc21 2024-11-15T17:41:41,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/916d26f11dd043578b09e7ff95005ceb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/916d26f11dd043578b09e7ff95005ceb 2024-11-15T17:41:41,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f7a79d407a4f4ded8abf357fdd2fe9d0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f7a79d407a4f4ded8abf357fdd2fe9d0 2024-11-15T17:41:41,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/86db8aa72d4b4229ad65833a1c739d86 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/86db8aa72d4b4229ad65833a1c739d86 2024-11-15T17:41:41,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/13ea0a8e12724e6286fb39f5c516692a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/13ea0a8e12724e6286fb39f5c516692a 2024-11-15T17:41:41,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e70e39e6ea7841c8a5df93fcc642a829 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e70e39e6ea7841c8a5df93fcc642a829 2024-11-15T17:41:42,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b7f583e68db34504862ba4b5b9b40060 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b7f583e68db34504862ba4b5b9b40060 2024-11-15T17:41:42,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/125fd34232b848d7b373b329d47a21e7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/125fd34232b848d7b373b329d47a21e7 2024-11-15T17:41:42,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/7573a1ba84a048da8796e59c6c4b9799 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/7573a1ba84a048da8796e59c6c4b9799 2024-11-15T17:41:42,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/11476806fb8e47ba916e495b0fc40791 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/11476806fb8e47ba916e495b0fc40791 2024-11-15T17:41:42,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/96ef35dad1cd47dd85cbfa1225df84a2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/96ef35dad1cd47dd85cbfa1225df84a2 2024-11-15T17:41:42,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e87bb71ed0984ea5a07985245f301a99 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/e87bb71ed0984ea5a07985245f301a99 2024-11-15T17:41:42,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/87cc50598e364942b430d732b1ed8745 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/87cc50598e364942b430d732b1ed8745 2024-11-15T17:41:42,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/d49a76035b1e40bdb9b620a70c857cb3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/d49a76035b1e40bdb9b620a70c857cb3 2024-11-15T17:41:42,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f1a22e2913043368fcffac075ae0238 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/6f1a22e2913043368fcffac075ae0238 2024-11-15T17:41:42,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/85acbfb3f1154e66ad518d18917d23ba to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/85acbfb3f1154e66ad518d18917d23ba 2024-11-15T17:41:42,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1ef77781f0d94712a55fbe8603c6e0db to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1ef77781f0d94712a55fbe8603c6e0db 2024-11-15T17:41:42,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f67b48bf1a364a7890e3cdd1dde22690 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/f67b48bf1a364a7890e3cdd1dde22690 2024-11-15T17:41:42,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/48b8af021b5849c987cdfa138a9e35d0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a440d91684bf450185345e6f4d4c4d50, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c15b7f7a9b3c41b5a3f5f5e5aa762df2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3b6aedda3869472bb5acc1936e494fb0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c676aa0b21e843ad87f866481a12ae10, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/b66055d929444b39bd018c38ce890970, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6745bb5d09cc4227a8ef81f5bb4689ee, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c227c7b0233943b1b6bc1e7aecf65d1b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/24276baec9914b558f88d43846cf6049, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/2cc497367e9c4323ab59cca71b842815, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9275bddf873f477e9c8d9c29a4067d2a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/e31f445dbc8b4a8aac9024386325dc4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3897d5b83abd4be2b4642c88180ab52a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/987abe95706b456697a3e9703e3ffbdc, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/78e972f79c034d26933f43bcdd2c7b69, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/d79de8df6bbe48259b05d40fbbb08c90, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/8efc6c91d7714a40a1547cd8b8b144bd, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5c263516334240cabcdf84d90617d6ac, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a90f44b8aba1453ab8a2c03790332dc6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6c06f2ffc54042e4b39cefd2c2f7a941, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/0b53a3db09c14388b8396b0a83553c2e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6fa6de1f51fc4fa2ba7e0a729ea78280, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/bb11acf82fab411bb8505f59c7196213, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/664a6887601f41f281bb62e5c61164a0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9f39659de27a46268af6c46b93f52f87, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/29bfeebd0bf94347bd45960a138cdc8e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/1e481d63379643e097ad261e412b8bc8] to archive 2024-11-15T17:41:42,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:41:42,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/48b8af021b5849c987cdfa138a9e35d0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/48b8af021b5849c987cdfa138a9e35d0 2024-11-15T17:41:42,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a440d91684bf450185345e6f4d4c4d50 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a440d91684bf450185345e6f4d4c4d50 2024-11-15T17:41:42,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c15b7f7a9b3c41b5a3f5f5e5aa762df2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c15b7f7a9b3c41b5a3f5f5e5aa762df2 2024-11-15T17:41:42,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3b6aedda3869472bb5acc1936e494fb0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3b6aedda3869472bb5acc1936e494fb0 2024-11-15T17:41:42,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c676aa0b21e843ad87f866481a12ae10 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c676aa0b21e843ad87f866481a12ae10 2024-11-15T17:41:42,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/b66055d929444b39bd018c38ce890970 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/b66055d929444b39bd018c38ce890970 2024-11-15T17:41:42,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6745bb5d09cc4227a8ef81f5bb4689ee to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6745bb5d09cc4227a8ef81f5bb4689ee 2024-11-15T17:41:42,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c227c7b0233943b1b6bc1e7aecf65d1b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/c227c7b0233943b1b6bc1e7aecf65d1b 2024-11-15T17:41:42,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/24276baec9914b558f88d43846cf6049 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/24276baec9914b558f88d43846cf6049 2024-11-15T17:41:42,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/2cc497367e9c4323ab59cca71b842815 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/2cc497367e9c4323ab59cca71b842815 2024-11-15T17:41:42,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9275bddf873f477e9c8d9c29a4067d2a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9275bddf873f477e9c8d9c29a4067d2a 2024-11-15T17:41:42,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/e31f445dbc8b4a8aac9024386325dc4e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/e31f445dbc8b4a8aac9024386325dc4e 2024-11-15T17:41:42,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3897d5b83abd4be2b4642c88180ab52a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/3897d5b83abd4be2b4642c88180ab52a 2024-11-15T17:41:42,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/987abe95706b456697a3e9703e3ffbdc to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/987abe95706b456697a3e9703e3ffbdc 2024-11-15T17:41:42,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/78e972f79c034d26933f43bcdd2c7b69 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/78e972f79c034d26933f43bcdd2c7b69 2024-11-15T17:41:42,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/d79de8df6bbe48259b05d40fbbb08c90 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/d79de8df6bbe48259b05d40fbbb08c90 2024-11-15T17:41:42,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/8efc6c91d7714a40a1547cd8b8b144bd to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/8efc6c91d7714a40a1547cd8b8b144bd 2024-11-15T17:41:42,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5c263516334240cabcdf84d90617d6ac to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5c263516334240cabcdf84d90617d6ac 2024-11-15T17:41:42,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a90f44b8aba1453ab8a2c03790332dc6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/a90f44b8aba1453ab8a2c03790332dc6 2024-11-15T17:41:42,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6c06f2ffc54042e4b39cefd2c2f7a941 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6c06f2ffc54042e4b39cefd2c2f7a941 2024-11-15T17:41:42,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/0b53a3db09c14388b8396b0a83553c2e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/0b53a3db09c14388b8396b0a83553c2e 2024-11-15T17:41:42,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6fa6de1f51fc4fa2ba7e0a729ea78280 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/6fa6de1f51fc4fa2ba7e0a729ea78280 2024-11-15T17:41:42,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/bb11acf82fab411bb8505f59c7196213 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/bb11acf82fab411bb8505f59c7196213 2024-11-15T17:41:42,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/664a6887601f41f281bb62e5c61164a0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/664a6887601f41f281bb62e5c61164a0 2024-11-15T17:41:42,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9f39659de27a46268af6c46b93f52f87 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/9f39659de27a46268af6c46b93f52f87 2024-11-15T17:41:42,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/29bfeebd0bf94347bd45960a138cdc8e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/29bfeebd0bf94347bd45960a138cdc8e 2024-11-15T17:41:42,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/1e481d63379643e097ad261e412b8bc8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/1e481d63379643e097ad261e412b8bc8 2024-11-15T17:41:42,056 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/recovered.edits/417.seqid, newMaxSeqId=417, maxSeqId=4 2024-11-15T17:41:42,057 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. 2024-11-15T17:41:42,057 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 057f1b8c8fc0614d025e1dac4cdaab16: Waiting for close lock at 1731692500260Running coprocessor pre-close hooks at 1731692500260Disabling compacts and flushes for region at 1731692500260Disabling writes for close at 1731692500260Obtaining lock to block concurrent updates at 1731692500261 (+1 ms)Preparing flush snapshotting stores in 057f1b8c8fc0614d025e1dac4cdaab16 at 1731692500261Finished memstore snapshotting TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731692500261Flushing stores of TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16. at 1731692500262 (+1 ms)Flushing 057f1b8c8fc0614d025e1dac4cdaab16/A: creating writer at 1731692500262Flushing mob file 057f1b8c8fc0614d025e1dac4cdaab16/A: appending metadata at 1731692500273 (+11 ms)Flushing mob file 057f1b8c8fc0614d025e1dac4cdaab16/A: closing flushed file at 1731692500273Flushing 057f1b8c8fc0614d025e1dac4cdaab16/A: appending metadata at 1731692500687 (+414 ms)Flushing 057f1b8c8fc0614d025e1dac4cdaab16/A: closing flushed file at 1731692500687Flushing 057f1b8c8fc0614d025e1dac4cdaab16/B: creating writer at 1731692501103 (+416 ms)Flushing 057f1b8c8fc0614d025e1dac4cdaab16/B: appending metadata at 1731692501106 (+3 ms)Flushing 057f1b8c8fc0614d025e1dac4cdaab16/B: closing flushed file at 1731692501106Flushing 057f1b8c8fc0614d025e1dac4cdaab16/C: creating writer at 1731692501518 (+412 ms)Flushing 057f1b8c8fc0614d025e1dac4cdaab16/C: appending metadata at 1731692501522 (+4 ms)Flushing 057f1b8c8fc0614d025e1dac4cdaab16/C: closing flushed file at 1731692501522Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9060c54: reopening flushed file at 1731692501930 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29eb398d: reopening flushed file at 1731692501935 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25709f76: reopening flushed file at 1731692501939 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 057f1b8c8fc0614d025e1dac4cdaab16 in 1683ms, sequenceid=414, compaction requested=true at 1731692501944 (+5 ms)Writing region close event to WAL at 1731692502052 (+108 ms)Running coprocessor post-close hooks at 1731692502056 (+4 ms)Closed at 1731692502057 (+1 ms) 2024-11-15T17:41:42,059 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=057f1b8c8fc0614d025e1dac4cdaab16, regionState=CLOSED 2024-11-15T17:41:42,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:41:42,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T17:41:42,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 057f1b8c8fc0614d025e1dac4cdaab16, server=4e3b17656000,45187,1731692383005 in 1.9560 sec 2024-11-15T17:41:42,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-15T17:41:42,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057f1b8c8fc0614d025e1dac4cdaab16, UNASSIGN in 1.9610 sec 2024-11-15T17:41:42,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T17:41:42,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9670 sec 2024-11-15T17:41:42,072 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692502072"}]},"ts":"1731692502072"} 2024-11-15T17:41:42,074 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:41:42,074 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:41:42,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9790 sec 2024-11-15T17:41:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T17:41:42,246 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:42,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:41:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:42,248 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T17:41:42,249 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:42,252 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,255 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/recovered.edits] 2024-11-15T17:41:42,259 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/08f8481335a644cc84fc1e108d1560e0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/08f8481335a644cc84fc1e108d1560e0 2024-11-15T17:41:42,261 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1067be3957a643ff86c8511580c71fcb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/1067be3957a643ff86c8511580c71fcb 2024-11-15T17:41:42,263 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5557683fc4144ec58610fcc0164830d1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/A/5557683fc4144ec58610fcc0164830d1 2024-11-15T17:41:42,266 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1883ece318934bd6b5b7e3ff6d215ae8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/1883ece318934bd6b5b7e3ff6d215ae8 2024-11-15T17:41:42,267 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/58a45f227a1e4fee8035befad0512f6a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/58a45f227a1e4fee8035befad0512f6a 2024-11-15T17:41:42,269 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b50345ed693d480f86c09d2e70aece1f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/B/b50345ed693d480f86c09d2e70aece1f 2024-11-15T17:41:42,271 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/47e2c21734494862baa8c062cdcd48a9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/47e2c21734494862baa8c062cdcd48a9 2024-11-15T17:41:42,272 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5fbc35ef74254ef68887c701f06502f4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/5fbc35ef74254ef68887c701f06502f4 2024-11-15T17:41:42,273 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/64e373216bec4d1e85082578a1795cc9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/C/64e373216bec4d1e85082578a1795cc9 2024-11-15T17:41:42,275 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/recovered.edits/417.seqid to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16/recovered.edits/417.seqid 2024-11-15T17:41:42,275 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,276 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:41:42,276 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:41:42,277 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T17:41:42,279 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150342e8a7d1d24df49f7586db8288281a_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150342e8a7d1d24df49f7586db8288281a_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,280 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d990831a5354146acaadc4bdbfbf059_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d990831a5354146acaadc4bdbfbf059_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,281 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515c007b1e22f42ffa76d5a9a5bcaddb9_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515c007b1e22f42ffa76d5a9a5bcaddb9_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,283 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515fcfafd22f54bf893e632c811837708_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515fcfafd22f54bf893e632c811837708_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,284 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151803ef8a127c42d49f1620f5a7748288_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151803ef8a127c42d49f1620f5a7748288_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,285 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151988433b5e45479ab2825acf64358fde_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151988433b5e45479ab2825acf64358fde_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,286 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115198b41fca3f842ac81f8ed850e4de781_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115198b41fca3f842ac81f8ed850e4de781_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,287 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115254af4c85ffd4acca32a625f1e5496f1_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115254af4c85ffd4acca32a625f1e5496f1_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,288 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152b70b737bebe490c8f8e536c2d18b083_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152b70b737bebe490c8f8e536c2d18b083_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,290 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111561d57e4ac68842e8b19d353a5d44582f_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111561d57e4ac68842e8b19d353a5d44582f_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,291 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563d71745ddba45849233d9446942b3f6_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563d71745ddba45849233d9446942b3f6_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,292 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111565c190f3e8b14911ad9f8c6bbe5c41a2_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111565c190f3e8b14911ad9f8c6bbe5c41a2_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,293 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111568d6815461414d77a2edd0e9f031ad61_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111568d6815461414d77a2edd0e9f031ad61_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,294 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a8fa9a215534bad8d75dcd57bcb31c8_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a8fa9a215534bad8d75dcd57bcb31c8_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,295 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111591e981160d244c1ca73998819dd63229_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111591e981160d244c1ca73998819dd63229_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,296 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c208e13ff6047f6b1db356d410f15dd_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c208e13ff6047f6b1db356d410f15dd_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,297 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c4ba038b0114f6bacac88d2b7c248e0_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c4ba038b0114f6bacac88d2b7c248e0_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,298 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f8f9bb5e49d494dac43afc4d949f957_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159f8f9bb5e49d494dac43afc4d949f957_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,299 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ca31d8557cab41e699e212dba61d8577_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ca31d8557cab41e699e212dba61d8577_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,300 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e606ba8b95f041f997145f1e83b5b129_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e606ba8b95f041f997145f1e83b5b129_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,301 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e753b002b4cf4ad0859e9d531d0202f7_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e753b002b4cf4ad0859e9d531d0202f7_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,301 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ece0b95efea44c6eb2a08dd57f334b65_057f1b8c8fc0614d025e1dac4cdaab16 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ece0b95efea44c6eb2a08dd57f334b65_057f1b8c8fc0614d025e1dac4cdaab16 2024-11-15T17:41:42,302 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:41:42,304 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:42,305 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:41:42,307 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:41:42,309 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:42,309 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:41:42,309 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731692502309"}]},"ts":"9223372036854775807"} 2024-11-15T17:41:42,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:41:42,311 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 057f1b8c8fc0614d025e1dac4cdaab16, NAME => 'TestAcidGuarantees,,1731692468445.057f1b8c8fc0614d025e1dac4cdaab16.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:41:42,311 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:41:42,311 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731692502311"}]},"ts":"9223372036854775807"} 2024-11-15T17:41:42,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:41:42,313 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:42,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-15T17:41:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T17:41:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:41:42,356 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:42,371 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 239), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 461), ProcessCount=11 (was 11), AvailableMemoryMB=9413 (was 8947) - AvailableMemoryMB LEAK? - 2024-11-15T17:41:42,381 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=9412 2024-11-15T17:41:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:41:42,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:41:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T17:41:42,384 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:41:42,384 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:42,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-15T17:41:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-15T17:41:42,385 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:41:42,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742291_1467 (size=959) 2024-11-15T17:41:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-15T17:41:42,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T17:41:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-15T17:41:42,796 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:41:42,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742292_1468 (size=53) 2024-11-15T17:41:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-15T17:41:43,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:41:43,205 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fbf7836fe88db5084a96e3c72d0c86aa, disabling compactions & flushes 2024-11-15T17:41:43,206 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:43,206 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:43,206 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. after waiting 0 ms 2024-11-15T17:41:43,206 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:43,206 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:43,206 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fbf7836fe88db5084a96e3c72d0c86aa: Waiting for close lock at 1731692503205Disabling compacts and flushes for region at 1731692503205Disabling writes for close at 1731692503206 (+1 ms)Writing region close event to WAL at 1731692503206Closed at 1731692503206 2024-11-15T17:41:43,208 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:41:43,209 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731692503208"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692503208"}]},"ts":"1731692503208"} 2024-11-15T17:41:43,212 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:41:43,214 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:41:43,215 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692503214"}]},"ts":"1731692503214"} 2024-11-15T17:41:43,217 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T17:41:43,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, ASSIGN}] 2024-11-15T17:41:43,218 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, ASSIGN 2024-11-15T17:41:43,219 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, ASSIGN; state=OFFLINE, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=false 2024-11-15T17:41:43,370 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=fbf7836fe88db5084a96e3c72d0c86aa, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:43,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, ASSIGN because future has completed 2024-11-15T17:41:43,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:41:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-15T17:41:43,533 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:43,533 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:41:43,536 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,536 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:41:43,536 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,536 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,539 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,540 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbf7836fe88db5084a96e3c72d0c86aa columnFamilyName A 2024-11-15T17:41:43,540 DEBUG [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:43,541 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] regionserver.HStore(327): Store=fbf7836fe88db5084a96e3c72d0c86aa/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:43,541 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,542 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbf7836fe88db5084a96e3c72d0c86aa columnFamilyName B 2024-11-15T17:41:43,542 DEBUG [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:43,542 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] regionserver.HStore(327): Store=fbf7836fe88db5084a96e3c72d0c86aa/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:43,542 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,543 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbf7836fe88db5084a96e3c72d0c86aa columnFamilyName C 2024-11-15T17:41:43,544 DEBUG [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:41:43,544 INFO [StoreOpener-fbf7836fe88db5084a96e3c72d0c86aa-1 {}] regionserver.HStore(327): Store=fbf7836fe88db5084a96e3c72d0c86aa/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:41:43,544 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,545 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,545 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,546 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,546 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,547 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:41:43,548 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,550 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:41:43,551 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened fbf7836fe88db5084a96e3c72d0c86aa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62869458, jitterRate=-0.06317207217216492}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:41:43,551 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:43,552 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for fbf7836fe88db5084a96e3c72d0c86aa: Running coprocessor pre-open hook at 1731692503536Writing region info on filesystem at 1731692503536Initializing all the Stores at 1731692503538 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692503538Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692503538Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692503538Cleaning up temporary data from old regions at 1731692503546 (+8 ms)Running coprocessor post-open hooks at 1731692503551 (+5 ms)Region opened successfully at 1731692503552 (+1 ms) 2024-11-15T17:41:43,553 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., pid=120, masterSystemTime=1731692503526 2024-11-15T17:41:43,555 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:43,555 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:43,556 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=fbf7836fe88db5084a96e3c72d0c86aa, regionState=OPEN, openSeqNum=2, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:41:43,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:41:43,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T17:41:43,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 in 185 msec 2024-11-15T17:41:43,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-15T17:41:43,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, ASSIGN in 344 msec 2024-11-15T17:41:43,565 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:41:43,565 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692503565"}]},"ts":"1731692503565"} 2024-11-15T17:41:43,567 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T17:41:43,568 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:41:43,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-15T17:41:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-15T17:41:44,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:44,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,538 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd4020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,540 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a6fd99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,541 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,543 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:41:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 4e3b17656000,41205,1731692382118 2024-11-15T17:41:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@565d91f8 2024-11-15T17:41:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:41:44,545 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:41:44,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,548 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5576aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,549 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da9b274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,551 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b13673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,552 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2136846c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,553 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185f2d0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7233de78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c22b9c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a654845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffd3713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c606733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,563 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626a8f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,564 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e7f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,566 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@83acaec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,567 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62588b78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44925ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,570 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@829a672, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,572 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44049499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,573 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75301166, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:41:44,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:41:44,575 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:41:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:41:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:41:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63897b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:41:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:41:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:41:44,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:41:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58432c3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:41:44,577 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,577 DEBUG [Thread-2089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,578 DEBUG [Thread-2102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,578 DEBUG [Thread-2087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,579 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,579 DEBUG [Thread-2106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-15T17:41:44,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T17:41:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,581 DEBUG [Thread-2093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,583 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:44,583 DEBUG [Thread-2093 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,584 DEBUG [Thread-2093 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:44,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:44,585 DEBUG [Thread-2095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,585 DEBUG [Thread-2091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,585 DEBUG [Thread-2104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,585 DEBUG [Thread-2100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:41:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:41:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:41:44,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:41:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2] 2024-11-15T17:41:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:44,595 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:44,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:41:44,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7ef164f975c64ea48992b431dc62ce00 is 50, key is test_row_0/A:col10/1731692504595/Put/seqid=0 2024-11-15T17:41:44,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742293_1469 (size=16681) 2024-11-15T17:41:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692564621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692564623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692564624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692564627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692564630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T17:41:44,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T17:41:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:44,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692564735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692564735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692564735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692564736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692564746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T17:41:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:44,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T17:41:44,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692564945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692564945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692564946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692564946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:44,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692564955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7ef164f975c64ea48992b431dc62ce00 2024-11-15T17:41:45,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/9a46f9daea5240c08762e3ff23a5cde6 is 50, key is test_row_0/B:col10/1731692504595/Put/seqid=0 2024-11-15T17:41:45,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T17:41:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:45,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:45,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742294_1470 (size=12001) 2024-11-15T17:41:45,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/9a46f9daea5240c08762e3ff23a5cde6 2024-11-15T17:41:45,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/34a20067c35a4c498e4696080a11bbaa is 50, key is test_row_0/C:col10/1731692504595/Put/seqid=0 2024-11-15T17:41:45,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742295_1471 (size=12001) 2024-11-15T17:41:45,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/34a20067c35a4c498e4696080a11bbaa 2024-11-15T17:41:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7ef164f975c64ea48992b431dc62ce00 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7ef164f975c64ea48992b431dc62ce00 2024-11-15T17:41:45,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7ef164f975c64ea48992b431dc62ce00, entries=250, sequenceid=14, filesize=16.3 K 2024-11-15T17:41:45,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/9a46f9daea5240c08762e3ff23a5cde6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/9a46f9daea5240c08762e3ff23a5cde6 2024-11-15T17:41:45,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/9a46f9daea5240c08762e3ff23a5cde6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T17:41:45,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/34a20067c35a4c498e4696080a11bbaa as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/34a20067c35a4c498e4696080a11bbaa 2024-11-15T17:41:45,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/34a20067c35a4c498e4696080a11bbaa, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T17:41:45,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fbf7836fe88db5084a96e3c72d0c86aa in 488ms, sequenceid=14, compaction requested=false 2024-11-15T17:41:45,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:45,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T17:41:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:45,198 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:41:45,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/d7adf642e1fa43bf99f10b59e0099951 is 50, key is test_row_0/A:col10/1731692504624/Put/seqid=0 2024-11-15T17:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T17:41:45,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742296_1472 (size=12001) 2024-11-15T17:41:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:45,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692565267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692565267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692565268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692565269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692565272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692565375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692565385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692565385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692565385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692565386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692565585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692565595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692565596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692565596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692565596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,607 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/d7adf642e1fa43bf99f10b59e0099951 2024-11-15T17:41:45,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/a9049a1134df47c1965904f0b483fcc5 is 50, key is test_row_0/B:col10/1731692504624/Put/seqid=0 2024-11-15T17:41:45,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742297_1473 (size=12001) 2024-11-15T17:41:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T17:41:45,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692565895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692565905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692565905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692565905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:45,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692565905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:46,017 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/a9049a1134df47c1965904f0b483fcc5 2024-11-15T17:41:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/20ef2e6861e44e13a33bc851bd1ce4e4 is 50, key is test_row_0/C:col10/1731692504624/Put/seqid=0 2024-11-15T17:41:46,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742298_1474 (size=12001) 2024-11-15T17:41:46,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692566405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:46,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692566416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:46,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692566416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:46,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692566416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:46,429 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/20ef2e6861e44e13a33bc851bd1ce4e4 2024-11-15T17:41:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692566425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:46,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/d7adf642e1fa43bf99f10b59e0099951 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/d7adf642e1fa43bf99f10b59e0099951 2024-11-15T17:41:46,435 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/d7adf642e1fa43bf99f10b59e0099951, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T17:41:46,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/a9049a1134df47c1965904f0b483fcc5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a9049a1134df47c1965904f0b483fcc5 2024-11-15T17:41:46,438 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a9049a1134df47c1965904f0b483fcc5, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T17:41:46,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/20ef2e6861e44e13a33bc851bd1ce4e4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/20ef2e6861e44e13a33bc851bd1ce4e4 2024-11-15T17:41:46,442 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/20ef2e6861e44e13a33bc851bd1ce4e4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T17:41:46,442 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for fbf7836fe88db5084a96e3c72d0c86aa in 1244ms, sequenceid=37, compaction requested=false 2024-11-15T17:41:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-15T17:41:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-15T17:41:46,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T17:41:46,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-15T17:41:46,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8660 sec 2024-11-15T17:41:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T17:41:46,726 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:46,726 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T17:41:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T17:41:46,727 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:46,728 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:46,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T17:41:46,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T17:41:46,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:46,880 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:41:46,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/73c5dda63ceb49f9b4d3f9b694a0b382 is 50, key is test_row_0/A:col10/1731692505271/Put/seqid=0 2024-11-15T17:41:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742299_1475 (size=12001) 2024-11-15T17:41:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T17:41:47,287 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/73c5dda63ceb49f9b4d3f9b694a0b382 2024-11-15T17:41:47,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/03f86364a1e5400f92ca87b0ff124fc1 is 50, key is test_row_0/B:col10/1731692505271/Put/seqid=0 2024-11-15T17:41:47,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742300_1476 (size=12001) 2024-11-15T17:41:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T17:41:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:47,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:47,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692567459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692567463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692567469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692567470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692567470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692567575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692567576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692567576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,581 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:41:47,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692567585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692567585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,697 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/03f86364a1e5400f92ca87b0ff124fc1 2024-11-15T17:41:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/17ca2ebfc6054046b2f7a9e7462975a4 is 50, key is test_row_0/C:col10/1731692505271/Put/seqid=0 2024-11-15T17:41:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742301_1477 (size=12001) 2024-11-15T17:41:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692567786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692567786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692567786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692567795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692567795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T17:41:48,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692568095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,106 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/17ca2ebfc6054046b2f7a9e7462975a4 2024-11-15T17:41:48,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/73c5dda63ceb49f9b4d3f9b694a0b382 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/73c5dda63ceb49f9b4d3f9b694a0b382 2024-11-15T17:41:48,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692568106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692568106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,113 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/73c5dda63ceb49f9b4d3f9b694a0b382, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T17:41:48,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/03f86364a1e5400f92ca87b0ff124fc1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/03f86364a1e5400f92ca87b0ff124fc1 2024-11-15T17:41:48,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692568115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692568115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,124 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/03f86364a1e5400f92ca87b0ff124fc1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T17:41:48,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/17ca2ebfc6054046b2f7a9e7462975a4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/17ca2ebfc6054046b2f7a9e7462975a4 2024-11-15T17:41:48,127 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/17ca2ebfc6054046b2f7a9e7462975a4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T17:41:48,128 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fbf7836fe88db5084a96e3c72d0c86aa in 1248ms, sequenceid=50, compaction requested=true 2024-11-15T17:41:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T17:41:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T17:41:48,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T17:41:48,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-15T17:41:48,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4050 sec 2024-11-15T17:41:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:48,620 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:48,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:41:48,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/234945c1dccd4275b7e076c663fd443e is 50, key is test_row_0/A:col10/1731692508619/Put/seqid=0 2024-11-15T17:41:48,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742302_1478 (size=14341) 2024-11-15T17:41:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692568633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692568634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692568635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692568635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692568635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692568745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692568745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692568745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692568756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T17:41:48,876 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:48,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T17:41:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:41:48,878 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:48,878 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:48,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:48,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692568955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692568955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692568955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692568966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:41:49,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/234945c1dccd4275b7e076c663fd443e 2024-11-15T17:41:49,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T17:41:49,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:49,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c0746bb17e074820a383b13645a98575 is 50, key is test_row_0/B:col10/1731692508619/Put/seqid=0 2024-11-15T17:41:49,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742303_1479 (size=12001) 2024-11-15T17:41:49,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T17:41:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:41:49,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692569265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692569265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692569276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692569276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T17:41:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:49,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c0746bb17e074820a383b13645a98575 2024-11-15T17:41:49,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/9459a0d6fa8841309063a48a2a6d2132 is 50, key is test_row_0/C:col10/1731692508619/Put/seqid=0 2024-11-15T17:41:49,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742304_1480 (size=12001) 2024-11-15T17:41:49,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/9459a0d6fa8841309063a48a2a6d2132 2024-11-15T17:41:49,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/234945c1dccd4275b7e076c663fd443e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/234945c1dccd4275b7e076c663fd443e 2024-11-15T17:41:49,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/234945c1dccd4275b7e076c663fd443e, entries=200, sequenceid=74, filesize=14.0 K 2024-11-15T17:41:49,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c0746bb17e074820a383b13645a98575 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0746bb17e074820a383b13645a98575 2024-11-15T17:41:49,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0746bb17e074820a383b13645a98575, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T17:41:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/9459a0d6fa8841309063a48a2a6d2132 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/9459a0d6fa8841309063a48a2a6d2132 2024-11-15T17:41:49,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/9459a0d6fa8841309063a48a2a6d2132, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T17:41:49,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fbf7836fe88db5084a96e3c72d0c86aa in 841ms, sequenceid=74, compaction requested=true 2024-11-15T17:41:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:49,461 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:49,461 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:49,462 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:49,462 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:49,462 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:41:49,462 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:41:49,462 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,462 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,462 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/9a46f9daea5240c08762e3ff23a5cde6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a9049a1134df47c1965904f0b483fcc5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/03f86364a1e5400f92ca87b0ff124fc1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0746bb17e074820a383b13645a98575] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=46.9 K 2024-11-15T17:41:49,462 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7ef164f975c64ea48992b431dc62ce00, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/d7adf642e1fa43bf99f10b59e0099951, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/73c5dda63ceb49f9b4d3f9b694a0b382, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/234945c1dccd4275b7e076c663fd443e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=53.7 K 2024-11-15T17:41:49,462 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a46f9daea5240c08762e3ff23a5cde6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731692504592 2024-11-15T17:41:49,462 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef164f975c64ea48992b431dc62ce00, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731692504586 2024-11-15T17:41:49,463 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting a9049a1134df47c1965904f0b483fcc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731692504620 2024-11-15T17:41:49,463 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7adf642e1fa43bf99f10b59e0099951, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731692504620 2024-11-15T17:41:49,463 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f86364a1e5400f92ca87b0ff124fc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731692505266 2024-11-15T17:41:49,463 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c5dda63ceb49f9b4d3f9b694a0b382, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731692505266 2024-11-15T17:41:49,463 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c0746bb17e074820a383b13645a98575, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731692507456 2024-11-15T17:41:49,463 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 234945c1dccd4275b7e076c663fd443e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731692507456 2024-11-15T17:41:49,469 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:49,470 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/956707a75381426982b45def124ba413 is 50, key is test_row_0/A:col10/1731692508619/Put/seqid=0 2024-11-15T17:41:49,472 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#397 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:49,472 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/ac785128a9e1492ebdb41fdc1c3711c5 is 50, key is test_row_0/B:col10/1731692508619/Put/seqid=0 2024-11-15T17:41:49,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742305_1481 (size=12139) 2024-11-15T17:41:49,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742306_1482 (size=12139) 2024-11-15T17:41:49,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T17:41:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,488 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:41:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/77aafeec4e1f4d62a6e6295f3c46d60b is 50, key is test_row_0/A:col10/1731692508632/Put/seqid=0 2024-11-15T17:41:49,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742307_1483 (size=12001) 2024-11-15T17:41:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:49,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:49,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692569789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692569795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692569797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692569799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692569799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,879 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/956707a75381426982b45def124ba413 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/956707a75381426982b45def124ba413 2024-11-15T17:41:49,883 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into 956707a75381426982b45def124ba413(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:49,883 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:49,883 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=12, startTime=1731692509461; duration=0sec 2024-11-15T17:41:49,883 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:49,884 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:41:49,884 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:49,885 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:49,885 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:41:49,885 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:49,885 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/34a20067c35a4c498e4696080a11bbaa, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/20ef2e6861e44e13a33bc851bd1ce4e4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/17ca2ebfc6054046b2f7a9e7462975a4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/9459a0d6fa8841309063a48a2a6d2132] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=46.9 K 2024-11-15T17:41:49,886 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34a20067c35a4c498e4696080a11bbaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731692504592 2024-11-15T17:41:49,888 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ef2e6861e44e13a33bc851bd1ce4e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731692504620 2024-11-15T17:41:49,888 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17ca2ebfc6054046b2f7a9e7462975a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731692505266 2024-11-15T17:41:49,889 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9459a0d6fa8841309063a48a2a6d2132, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731692507456 2024-11-15T17:41:49,889 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/ac785128a9e1492ebdb41fdc1c3711c5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/ac785128a9e1492ebdb41fdc1c3711c5 2024-11-15T17:41:49,893 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into ac785128a9e1492ebdb41fdc1c3711c5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:49,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:49,893 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=12, startTime=1731692509461; duration=0sec 2024-11-15T17:41:49,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:49,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:41:49,897 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:49,897 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/58b88538278b4fe982f6f4ce6f7e7d68 is 50, key is test_row_0/C:col10/1731692508619/Put/seqid=0 2024-11-15T17:41:49,900 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/77aafeec4e1f4d62a6e6295f3c46d60b 2024-11-15T17:41:49,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742308_1484 (size=12139) 2024-11-15T17:41:49,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/d975774ba2ee4bb2aa3588ae876f4f8c is 50, key is test_row_0/B:col10/1731692508632/Put/seqid=0 2024-11-15T17:41:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692569905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692569905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742309_1485 (size=12001) 2024-11-15T17:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692569916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:49,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692569926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:41:50,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692570115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692570115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692570126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692570145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,306 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/58b88538278b4fe982f6f4ce6f7e7d68 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/58b88538278b4fe982f6f4ce6f7e7d68 2024-11-15T17:41:50,310 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into 58b88538278b4fe982f6f4ce6f7e7d68(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:50,310 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:50,310 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/d975774ba2ee4bb2aa3588ae876f4f8c 2024-11-15T17:41:50,310 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=12, startTime=1731692509461; duration=0sec 2024-11-15T17:41:50,310 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:50,310 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:41:50,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/c2fa5c057e894b33a9f171c0fe98ab59 is 50, key is test_row_0/C:col10/1731692508632/Put/seqid=0 2024-11-15T17:41:50,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742310_1486 (size=12001) 2024-11-15T17:41:50,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692570425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692570435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692570446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692570456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,725 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/c2fa5c057e894b33a9f171c0fe98ab59 2024-11-15T17:41:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/77aafeec4e1f4d62a6e6295f3c46d60b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/77aafeec4e1f4d62a6e6295f3c46d60b 2024-11-15T17:41:50,731 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/77aafeec4e1f4d62a6e6295f3c46d60b, entries=150, sequenceid=86, filesize=11.7 K 2024-11-15T17:41:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/d975774ba2ee4bb2aa3588ae876f4f8c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d975774ba2ee4bb2aa3588ae876f4f8c 2024-11-15T17:41:50,735 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d975774ba2ee4bb2aa3588ae876f4f8c, entries=150, sequenceid=86, filesize=11.7 K 2024-11-15T17:41:50,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/c2fa5c057e894b33a9f171c0fe98ab59 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/c2fa5c057e894b33a9f171c0fe98ab59 2024-11-15T17:41:50,738 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/c2fa5c057e894b33a9f171c0fe98ab59, entries=150, sequenceid=86, filesize=11.7 K 2024-11-15T17:41:50,739 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for fbf7836fe88db5084a96e3c72d0c86aa in 1251ms, sequenceid=86, compaction requested=false 2024-11-15T17:41:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T17:41:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T17:41:50,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T17:41:50,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-11-15T17:41:50,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8680 sec 2024-11-15T17:41:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:50,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:50,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T17:41:50,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c58fe9f0f57b402199485cbb215d45f5 is 50, key is test_row_0/A:col10/1731692510819/Put/seqid=0 2024-11-15T17:41:50,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742311_1487 (size=14341) 2024-11-15T17:41:50,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692570858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692570935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692570956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692570956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692570965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:50,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692570975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T17:41:51,026 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:51,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T17:41:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:41:51,028 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:51,028 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:51,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:41:51,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:41:51,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:51,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692571186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:51,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c58fe9f0f57b402199485cbb215d45f5 2024-11-15T17:41:51,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/6119f8b2a0f5497da75d77b3b07e9d1c is 50, key is test_row_0/B:col10/1731692510819/Put/seqid=0 2024-11-15T17:41:51,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742312_1488 (size=12001) 2024-11-15T17:41:51,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:41:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:41:51,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:41:51,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:51,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692571495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:51,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:41:51,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:51,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/6119f8b2a0f5497da75d77b3b07e9d1c 2024-11-15T17:41:51,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/137295b3308e45c594b01ad22b4c2f4e is 50, key is test_row_0/C:col10/1731692510819/Put/seqid=0 2024-11-15T17:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:41:51,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742313_1489 (size=12001) 2024-11-15T17:41:51,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:41:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:51,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:41:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:51,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:51,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692571955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:51,960 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692571965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:51,971 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:51,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692571975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:51,979 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:51,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:51,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692571986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:51,987 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:52,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692572006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:52,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/137295b3308e45c594b01ad22b4c2f4e 2024-11-15T17:41:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c58fe9f0f57b402199485cbb215d45f5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c58fe9f0f57b402199485cbb215d45f5 2024-11-15T17:41:52,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c58fe9f0f57b402199485cbb215d45f5, entries=200, sequenceid=115, filesize=14.0 K 2024-11-15T17:41:52,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/6119f8b2a0f5497da75d77b3b07e9d1c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/6119f8b2a0f5497da75d77b3b07e9d1c 2024-11-15T17:41:52,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/6119f8b2a0f5497da75d77b3b07e9d1c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T17:41:52,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/137295b3308e45c594b01ad22b4c2f4e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/137295b3308e45c594b01ad22b4c2f4e 2024-11-15T17:41:52,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/137295b3308e45c594b01ad22b4c2f4e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T17:41:52,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for fbf7836fe88db5084a96e3c72d0c86aa in 1256ms, sequenceid=115, compaction requested=true 2024-11-15T17:41:52,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:52,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:52,075 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:52,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:52,076 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:52,076 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:52,076 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:52,076 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:41:52,076 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:41:52,076 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:52,076 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:52,076 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/956707a75381426982b45def124ba413, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/77aafeec4e1f4d62a6e6295f3c46d60b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c58fe9f0f57b402199485cbb215d45f5] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=37.6 K 2024-11-15T17:41:52,076 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/ac785128a9e1492ebdb41fdc1c3711c5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d975774ba2ee4bb2aa3588ae876f4f8c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/6119f8b2a0f5497da75d77b3b07e9d1c] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=35.3 K 2024-11-15T17:41:52,077 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 956707a75381426982b45def124ba413, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731692507456 2024-11-15T17:41:52,077 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting ac785128a9e1492ebdb41fdc1c3711c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731692507456 2024-11-15T17:41:52,077 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d975774ba2ee4bb2aa3588ae876f4f8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731692508632 2024-11-15T17:41:52,077 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77aafeec4e1f4d62a6e6295f3c46d60b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731692508632 2024-11-15T17:41:52,077 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c58fe9f0f57b402199485cbb215d45f5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731692509780 2024-11-15T17:41:52,077 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6119f8b2a0f5497da75d77b3b07e9d1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731692509780 2024-11-15T17:41:52,082 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:52,083 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/6627b321ef2a487dabbb81ae8092672d is 50, key is test_row_0/A:col10/1731692510819/Put/seqid=0 2024-11-15T17:41:52,085 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:52,085 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/fcc3d87df37d46dda31555397b537555 is 50, key is test_row_0/B:col10/1731692510819/Put/seqid=0 2024-11-15T17:41:52,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742314_1490 (size=12241) 2024-11-15T17:41:52,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742315_1491 (size=12241) 2024-11-15T17:41:52,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T17:41:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:52,096 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T17:41:52,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a02e4387ff8e410b9f2670b72df10f21 is 50, key is test_row_0/A:col10/1731692510830/Put/seqid=0 2024-11-15T17:41:52,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742316_1492 (size=12001) 2024-11-15T17:41:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:41:52,491 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/6627b321ef2a487dabbb81ae8092672d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6627b321ef2a487dabbb81ae8092672d 2024-11-15T17:41:52,494 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/fcc3d87df37d46dda31555397b537555 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/fcc3d87df37d46dda31555397b537555 2024-11-15T17:41:52,494 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into 6627b321ef2a487dabbb81ae8092672d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:52,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:52,494 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=13, startTime=1731692512075; duration=0sec 2024-11-15T17:41:52,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:52,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:41:52,494 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:52,503 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:52,503 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:41:52,503 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:52,503 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/58b88538278b4fe982f6f4ce6f7e7d68, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/c2fa5c057e894b33a9f171c0fe98ab59, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/137295b3308e45c594b01ad22b4c2f4e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=35.3 K 2024-11-15T17:41:52,503 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b88538278b4fe982f6f4ce6f7e7d68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731692507456 2024-11-15T17:41:52,503 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2fa5c057e894b33a9f171c0fe98ab59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731692508632 2024-11-15T17:41:52,504 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137295b3308e45c594b01ad22b4c2f4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731692509780 2024-11-15T17:41:52,505 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into fcc3d87df37d46dda31555397b537555(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:52,505 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:52,505 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=13, startTime=1731692512075; duration=0sec 2024-11-15T17:41:52,506 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:52,506 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:41:52,509 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a02e4387ff8e410b9f2670b72df10f21 2024-11-15T17:41:52,510 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:52,511 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/bbe5048741ec46dbb8317c0fa03dc2a5 is 50, key is test_row_0/C:col10/1731692510819/Put/seqid=0 2024-11-15T17:41:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/49cda229dd3d49aea6b305f33e8824af is 50, key is test_row_0/B:col10/1731692510830/Put/seqid=0 2024-11-15T17:41:52,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742317_1493 (size=12241) 2024-11-15T17:41:52,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742318_1494 (size=12001) 2024-11-15T17:41:52,533 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/bbe5048741ec46dbb8317c0fa03dc2a5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bbe5048741ec46dbb8317c0fa03dc2a5 2024-11-15T17:41:52,539 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into bbe5048741ec46dbb8317c0fa03dc2a5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:52,539 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:52,539 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=13, startTime=1731692512076; duration=0sec 2024-11-15T17:41:52,539 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:52,539 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:41:52,540 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T17:41:52,540 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T17:41:52,930 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/49cda229dd3d49aea6b305f33e8824af 2024-11-15T17:41:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/8a7e8c1bbdc247d7afe403dec5f55966 is 50, key is test_row_0/C:col10/1731692510830/Put/seqid=0 2024-11-15T17:41:52,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742319_1495 (size=12001) 2024-11-15T17:41:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:53,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692573143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:41:53,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692573255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:53,338 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/8a7e8c1bbdc247d7afe403dec5f55966 2024-11-15T17:41:53,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a02e4387ff8e410b9f2670b72df10f21 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a02e4387ff8e410b9f2670b72df10f21 2024-11-15T17:41:53,344 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a02e4387ff8e410b9f2670b72df10f21, entries=150, sequenceid=125, filesize=11.7 K 2024-11-15T17:41:53,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/49cda229dd3d49aea6b305f33e8824af as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/49cda229dd3d49aea6b305f33e8824af 2024-11-15T17:41:53,348 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/49cda229dd3d49aea6b305f33e8824af, entries=150, sequenceid=125, filesize=11.7 K 2024-11-15T17:41:53,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/8a7e8c1bbdc247d7afe403dec5f55966 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/8a7e8c1bbdc247d7afe403dec5f55966 2024-11-15T17:41:53,351 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/8a7e8c1bbdc247d7afe403dec5f55966, entries=150, sequenceid=125, filesize=11.7 K 2024-11-15T17:41:53,352 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for fbf7836fe88db5084a96e3c72d0c86aa in 1256ms, sequenceid=125, compaction requested=false 2024-11-15T17:41:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T17:41:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T17:41:53,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T17:41:53,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3250 sec 2024-11-15T17:41:53,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3290 sec 2024-11-15T17:41:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:53,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T17:41:53,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/9a64d36af302403da1121df65fa7f6c2 is 50, key is test_row_0/A:col10/1731692513133/Put/seqid=0 2024-11-15T17:41:53,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742320_1496 (size=14541) 2024-11-15T17:41:53,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692573497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:53,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692573605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:53,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692573815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:53,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/9a64d36af302403da1121df65fa7f6c2 2024-11-15T17:41:53,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/73a6e86a679042c1b3b6483dc8ceece0 is 50, key is test_row_0/B:col10/1731692513133/Put/seqid=0 2024-11-15T17:41:53,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742321_1497 (size=12151) 2024-11-15T17:41:53,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/73a6e86a679042c1b3b6483dc8ceece0 2024-11-15T17:41:53,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/507ceadd168c41209cb07d087510f8de is 50, key is test_row_0/C:col10/1731692513133/Put/seqid=0 2024-11-15T17:41:53,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742322_1498 (size=12151) 2024-11-15T17:41:53,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692573976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,980 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:53,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692573985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,990 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692573985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:53,992 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692573995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:54,001 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:41:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:54,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692574125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:54,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/507ceadd168c41209cb07d087510f8de 2024-11-15T17:41:54,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/9a64d36af302403da1121df65fa7f6c2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9a64d36af302403da1121df65fa7f6c2 2024-11-15T17:41:54,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9a64d36af302403da1121df65fa7f6c2, entries=200, sequenceid=155, filesize=14.2 K 2024-11-15T17:41:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/73a6e86a679042c1b3b6483dc8ceece0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/73a6e86a679042c1b3b6483dc8ceece0 2024-11-15T17:41:54,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/73a6e86a679042c1b3b6483dc8ceece0, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T17:41:54,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/507ceadd168c41209cb07d087510f8de as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/507ceadd168c41209cb07d087510f8de 2024-11-15T17:41:54,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/507ceadd168c41209cb07d087510f8de, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T17:41:54,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for fbf7836fe88db5084a96e3c72d0c86aa in 840ms, sequenceid=155, compaction requested=true 2024-11-15T17:41:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:54,312 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:54,312 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:54,312 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:54,312 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:54,312 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:41:54,312 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:41:54,313 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:54,313 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:54,313 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/fcc3d87df37d46dda31555397b537555, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/49cda229dd3d49aea6b305f33e8824af, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/73a6e86a679042c1b3b6483dc8ceece0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=35.5 K 2024-11-15T17:41:54,313 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6627b321ef2a487dabbb81ae8092672d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a02e4387ff8e410b9f2670b72df10f21, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9a64d36af302403da1121df65fa7f6c2] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=37.9 K 2024-11-15T17:41:54,313 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc3d87df37d46dda31555397b537555, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731692509780 2024-11-15T17:41:54,313 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6627b321ef2a487dabbb81ae8092672d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731692509780 2024-11-15T17:41:54,313 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 49cda229dd3d49aea6b305f33e8824af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731692510830 2024-11-15T17:41:54,313 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting a02e4387ff8e410b9f2670b72df10f21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731692510830 2024-11-15T17:41:54,313 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a6e86a679042c1b3b6483dc8ceece0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692513088 2024-11-15T17:41:54,313 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a64d36af302403da1121df65fa7f6c2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692513088 2024-11-15T17:41:54,328 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:54,328 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:54,328 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c901f585555548bd8c75a8ce2eb0a00e is 50, key is test_row_0/A:col10/1731692513133/Put/seqid=0 2024-11-15T17:41:54,329 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/7bc06f927afa4647aca289e87544f845 is 50, key is test_row_0/B:col10/1731692513133/Put/seqid=0 2024-11-15T17:41:54,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742324_1500 (size=12493) 2024-11-15T17:41:54,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742323_1499 (size=12493) 2024-11-15T17:41:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:54,642 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:54,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:41:54,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7b01b3fbbce54cd6b63e33496c9c572c is 50, key is test_row_0/A:col10/1731692514638/Put/seqid=0 2024-11-15T17:41:54,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742325_1501 (size=16931) 2024-11-15T17:41:54,735 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c901f585555548bd8c75a8ce2eb0a00e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c901f585555548bd8c75a8ce2eb0a00e 2024-11-15T17:41:54,735 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/7bc06f927afa4647aca289e87544f845 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/7bc06f927afa4647aca289e87544f845 2024-11-15T17:41:54,739 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into 7bc06f927afa4647aca289e87544f845(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:54,739 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:54,739 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=13, startTime=1731692514312; duration=0sec 2024-11-15T17:41:54,739 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:54,739 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:41:54,739 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:54,739 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into c901f585555548bd8c75a8ce2eb0a00e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:54,739 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:54,739 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=13, startTime=1731692514312; duration=0sec 2024-11-15T17:41:54,739 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:54,739 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:41:54,740 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:54,740 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:41:54,740 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:54,740 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bbe5048741ec46dbb8317c0fa03dc2a5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/8a7e8c1bbdc247d7afe403dec5f55966, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/507ceadd168c41209cb07d087510f8de] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=35.5 K 2024-11-15T17:41:54,740 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe5048741ec46dbb8317c0fa03dc2a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731692509780 2024-11-15T17:41:54,741 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7e8c1bbdc247d7afe403dec5f55966, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731692510830 2024-11-15T17:41:54,741 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 507ceadd168c41209cb07d087510f8de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692513088 2024-11-15T17:41:54,746 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:54,747 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/bb862a4d1c034565ac8ef644c40493c7 is 50, key is test_row_0/C:col10/1731692513133/Put/seqid=0 2024-11-15T17:41:54,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742326_1502 (size=12493) 2024-11-15T17:41:54,753 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/bb862a4d1c034565ac8ef644c40493c7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bb862a4d1c034565ac8ef644c40493c7 2024-11-15T17:41:54,757 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into bb862a4d1c034565ac8ef644c40493c7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:54,757 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:54,757 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=13, startTime=1731692514312; duration=0sec 2024-11-15T17:41:54,757 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:54,757 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:41:54,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692574777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:54,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692574895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:55,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7b01b3fbbce54cd6b63e33496c9c572c 2024-11-15T17:41:55,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/5887f85457d14f9db31048efe6b49768 is 50, key is test_row_0/B:col10/1731692514638/Put/seqid=0 2024-11-15T17:41:55,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742327_1503 (size=12151) 2024-11-15T17:41:55,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692575105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T17:41:55,196 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:55,196 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T17:41:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:41:55,198 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:55,198 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:55,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:41:55,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T17:41:55,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:55,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:55,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:55,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:55,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:55,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692575415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:55,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/5887f85457d14f9db31048efe6b49768 2024-11-15T17:41:55,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/733556605fdd4f5da8b69582ce7a8d29 is 50, key is test_row_0/C:col10/1731692514638/Put/seqid=0 2024-11-15T17:41:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742328_1504 (size=12151) 2024-11-15T17:41:55,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/733556605fdd4f5da8b69582ce7a8d29 2024-11-15T17:41:55,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7b01b3fbbce54cd6b63e33496c9c572c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7b01b3fbbce54cd6b63e33496c9c572c 2024-11-15T17:41:55,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7b01b3fbbce54cd6b63e33496c9c572c, entries=250, sequenceid=166, filesize=16.5 K 2024-11-15T17:41:55,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/5887f85457d14f9db31048efe6b49768 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/5887f85457d14f9db31048efe6b49768 2024-11-15T17:41:55,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/5887f85457d14f9db31048efe6b49768, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T17:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/733556605fdd4f5da8b69582ce7a8d29 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/733556605fdd4f5da8b69582ce7a8d29 2024-11-15T17:41:55,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/733556605fdd4f5da8b69582ce7a8d29, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T17:41:55,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fbf7836fe88db5084a96e3c72d0c86aa in 854ms, sequenceid=166, compaction requested=false 2024-11-15T17:41:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:55,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T17:41:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:55,503 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:41:55,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/9650063d8bf242eca9c9f009bdb8000e is 50, key is test_row_0/A:col10/1731692514773/Put/seqid=0 2024-11-15T17:41:55,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742329_1505 (size=12151) 2024-11-15T17:41:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:41:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:41:55,912 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/9650063d8bf242eca9c9f009bdb8000e 2024-11-15T17:41:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/d3fbc31ce3a449129e5069949b6561eb is 50, key is test_row_0/B:col10/1731692514773/Put/seqid=0 2024-11-15T17:41:55,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742330_1506 (size=12151) 2024-11-15T17:41:55,927 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/d3fbc31ce3a449129e5069949b6561eb 2024-11-15T17:41:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:55,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:55,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/87e16ed10021424796573e8e31be652e is 50, key is test_row_0/C:col10/1731692514773/Put/seqid=0 2024-11-15T17:41:55,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742331_1507 (size=12151) 2024-11-15T17:41:55,940 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/87e16ed10021424796573e8e31be652e 2024-11-15T17:41:55,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/9650063d8bf242eca9c9f009bdb8000e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9650063d8bf242eca9c9f009bdb8000e 2024-11-15T17:41:55,947 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9650063d8bf242eca9c9f009bdb8000e, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T17:41:55,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/d3fbc31ce3a449129e5069949b6561eb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d3fbc31ce3a449129e5069949b6561eb 2024-11-15T17:41:55,952 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d3fbc31ce3a449129e5069949b6561eb, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T17:41:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/87e16ed10021424796573e8e31be652e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/87e16ed10021424796573e8e31be652e 2024-11-15T17:41:55,956 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/87e16ed10021424796573e8e31be652e, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T17:41:55,957 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for fbf7836fe88db5084a96e3c72d0c86aa in 454ms, sequenceid=194, compaction requested=true 2024-11-15T17:41:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T17:41:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T17:41:55,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T17:41:55,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 761 msec 2024-11-15T17:41:55,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 768 msec 2024-11-15T17:41:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:55,970 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:55,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:41:55,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/49b7643bf3ca447ba4bc59f0906a898b is 50, key is test_row_0/A:col10/1731692515969/Put/seqid=0 2024-11-15T17:41:55,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742332_1508 (size=12147) 2024-11-15T17:41:56,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692576117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:56,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692576225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T17:41:56,336 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:56,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T17:41:56,337 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:41:56,338 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:56,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:56,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/49b7643bf3ca447ba4bc59f0906a898b 2024-11-15T17:41:56,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/41db1e3e24a2456e8bd7a46a43981629 is 50, key is test_row_0/B:col10/1731692515969/Put/seqid=0 2024-11-15T17:41:56,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742333_1509 (size=9757) 2024-11-15T17:41:56,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692576435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:41:56,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:41:56,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:56,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:41:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:41:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692576745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:56,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/41db1e3e24a2456e8bd7a46a43981629 2024-11-15T17:41:56,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/b70294b0f32a45acba0a99b8eddaeb9e is 50, key is test_row_0/C:col10/1731692515969/Put/seqid=0 2024-11-15T17:41:56,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:41:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742334_1510 (size=9757) 2024-11-15T17:41:56,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:41:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:56,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:41:57,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:41:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/b70294b0f32a45acba0a99b8eddaeb9e 2024-11-15T17:41:57,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/49b7643bf3ca447ba4bc59f0906a898b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/49b7643bf3ca447ba4bc59f0906a898b 2024-11-15T17:41:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/49b7643bf3ca447ba4bc59f0906a898b, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T17:41:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/41db1e3e24a2456e8bd7a46a43981629 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/41db1e3e24a2456e8bd7a46a43981629 2024-11-15T17:41:57,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/41db1e3e24a2456e8bd7a46a43981629, entries=100, sequenceid=205, filesize=9.5 K 2024-11-15T17:41:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/b70294b0f32a45acba0a99b8eddaeb9e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b70294b0f32a45acba0a99b8eddaeb9e 2024-11-15T17:41:57,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b70294b0f32a45acba0a99b8eddaeb9e, entries=100, sequenceid=205, filesize=9.5 K 2024-11-15T17:41:57,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for fbf7836fe88db5084a96e3c72d0c86aa in 1242ms, sequenceid=205, compaction requested=true 2024-11-15T17:41:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:57,212 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:57,212 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:57,213 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:57,213 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:57,213 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:41:57,213 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:41:57,213 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,213 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,213 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/7bc06f927afa4647aca289e87544f845, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/5887f85457d14f9db31048efe6b49768, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d3fbc31ce3a449129e5069949b6561eb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/41db1e3e24a2456e8bd7a46a43981629] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=45.5 K 2024-11-15T17:41:57,213 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c901f585555548bd8c75a8ce2eb0a00e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7b01b3fbbce54cd6b63e33496c9c572c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9650063d8bf242eca9c9f009bdb8000e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/49b7643bf3ca447ba4bc59f0906a898b] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=52.5 K 2024-11-15T17:41:57,213 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bc06f927afa4647aca289e87544f845, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692513088 2024-11-15T17:41:57,213 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c901f585555548bd8c75a8ce2eb0a00e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692513088 2024-11-15T17:41:57,214 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 5887f85457d14f9db31048efe6b49768, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692513473 2024-11-15T17:41:57,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b01b3fbbce54cd6b63e33496c9c572c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692513473 2024-11-15T17:41:57,214 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d3fbc31ce3a449129e5069949b6561eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731692514762 2024-11-15T17:41:57,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9650063d8bf242eca9c9f009bdb8000e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731692514762 2024-11-15T17:41:57,214 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 41db1e3e24a2456e8bd7a46a43981629, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692515961 2024-11-15T17:41:57,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b7643bf3ca447ba4bc59f0906a898b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692515961 2024-11-15T17:41:57,219 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:57,220 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/bb67de5adbf14abaaf16d50cd6fd4fe2 is 50, key is test_row_0/B:col10/1731692515969/Put/seqid=0 2024-11-15T17:41:57,221 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:57,221 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7524742cffd34dcb8ac5954f41566cf1 is 50, key is test_row_0/A:col10/1731692515969/Put/seqid=0 2024-11-15T17:41:57,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742335_1511 (size=12629) 2024-11-15T17:41:57,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742336_1512 (size=12629) 2024-11-15T17:41:57,238 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/bb67de5adbf14abaaf16d50cd6fd4fe2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/bb67de5adbf14abaaf16d50cd6fd4fe2 2024-11-15T17:41:57,240 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7524742cffd34dcb8ac5954f41566cf1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7524742cffd34dcb8ac5954f41566cf1 2024-11-15T17:41:57,242 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into bb67de5adbf14abaaf16d50cd6fd4fe2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:57,242 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,242 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=12, startTime=1731692517212; duration=0sec 2024-11-15T17:41:57,243 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:57,243 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:41:57,243 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:41:57,244 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:41:57,244 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:41:57,244 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,244 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bb862a4d1c034565ac8ef644c40493c7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/733556605fdd4f5da8b69582ce7a8d29, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/87e16ed10021424796573e8e31be652e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b70294b0f32a45acba0a99b8eddaeb9e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=45.5 K 2024-11-15T17:41:57,245 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bb862a4d1c034565ac8ef644c40493c7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692513088 2024-11-15T17:41:57,245 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 733556605fdd4f5da8b69582ce7a8d29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731692513473 2024-11-15T17:41:57,245 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e16ed10021424796573e8e31be652e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731692514762 2024-11-15T17:41:57,246 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b70294b0f32a45acba0a99b8eddaeb9e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692515961 2024-11-15T17:41:57,246 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into 7524742cffd34dcb8ac5954f41566cf1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:57,246 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,246 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=12, startTime=1731692517212; duration=0sec 2024-11-15T17:41:57,247 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:57,247 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:41:57,252 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:57,252 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/2ea9029244214f5a8cd1ae979d018182 is 50, key is test_row_0/C:col10/1731692515969/Put/seqid=0 2024-11-15T17:41:57,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T17:41:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,254 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:41:57,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c51636f7406a4cc49ca78b6b0e661283 is 50, key is test_row_0/A:col10/1731692516103/Put/seqid=0 2024-11-15T17:41:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:57,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:57,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742337_1513 (size=12151) 2024-11-15T17:41:57,271 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c51636f7406a4cc49ca78b6b0e661283 2024-11-15T17:41:57,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742338_1514 (size=12629) 2024-11-15T17:41:57,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/08f050eb35a84a6a8359273207c9de69 is 50, key is test_row_0/B:col10/1731692516103/Put/seqid=0 2024-11-15T17:41:57,280 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/2ea9029244214f5a8cd1ae979d018182 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/2ea9029244214f5a8cd1ae979d018182 2024-11-15T17:41:57,286 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into 2ea9029244214f5a8cd1ae979d018182(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:57,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,286 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=12, startTime=1731692517212; duration=0sec 2024-11-15T17:41:57,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:57,286 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:41:57,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742339_1515 (size=12151) 2024-11-15T17:41:57,294 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/08f050eb35a84a6a8359273207c9de69 2024-11-15T17:41:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/0a30ca983f514687a0ff08330d1a65cb is 50, key is test_row_0/C:col10/1731692516103/Put/seqid=0 2024-11-15T17:41:57,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742340_1516 (size=12151) 2024-11-15T17:41:57,308 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/0a30ca983f514687a0ff08330d1a65cb 2024-11-15T17:41:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c51636f7406a4cc49ca78b6b0e661283 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c51636f7406a4cc49ca78b6b0e661283 2024-11-15T17:41:57,317 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c51636f7406a4cc49ca78b6b0e661283, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T17:41:57,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/08f050eb35a84a6a8359273207c9de69 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/08f050eb35a84a6a8359273207c9de69 2024-11-15T17:41:57,321 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/08f050eb35a84a6a8359273207c9de69, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T17:41:57,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/0a30ca983f514687a0ff08330d1a65cb as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/0a30ca983f514687a0ff08330d1a65cb 2024-11-15T17:41:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692577317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:57,327 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/0a30ca983f514687a0ff08330d1a65cb, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T17:41:57,327 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fbf7836fe88db5084a96e3c72d0c86aa in 74ms, sequenceid=232, compaction requested=false 2024-11-15T17:41:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T17:41:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T17:41:57,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T17:41:57,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-15T17:41:57,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 995 msec 2024-11-15T17:41:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:57,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:41:57,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7bb322c6ae0e45ee8af804f7467a7ae1 is 50, key is test_row_0/A:col10/1731692517314/Put/seqid=0 2024-11-15T17:41:57,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742341_1517 (size=14541) 2024-11-15T17:41:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T17:41:57,476 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:57,476 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-15T17:41:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:41:57,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:57,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:57,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:41:57,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692577580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:57,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T17:41:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692577695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:57,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T17:41:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:41:57,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7bb322c6ae0e45ee8af804f7467a7ae1 2024-11-15T17:41:57,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/eb548c91247d4421bcc94d0c2ed9777b is 50, key is test_row_0/B:col10/1731692517314/Put/seqid=0 2024-11-15T17:41:57,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742342_1518 (size=12151) 2024-11-15T17:41:57,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/eb548c91247d4421bcc94d0c2ed9777b 2024-11-15T17:41:57,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/5f5896542231473180cc7253744ba31d is 50, key is test_row_0/C:col10/1731692517314/Put/seqid=0 2024-11-15T17:41:57,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742343_1519 (size=12151) 2024-11-15T17:41:57,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/5f5896542231473180cc7253744ba31d 2024-11-15T17:41:57,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/7bb322c6ae0e45ee8af804f7467a7ae1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7bb322c6ae0e45ee8af804f7467a7ae1 2024-11-15T17:41:57,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7bb322c6ae0e45ee8af804f7467a7ae1, entries=200, sequenceid=245, filesize=14.2 K 2024-11-15T17:41:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/eb548c91247d4421bcc94d0c2ed9777b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eb548c91247d4421bcc94d0c2ed9777b 2024-11-15T17:41:57,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eb548c91247d4421bcc94d0c2ed9777b, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T17:41:57,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/5f5896542231473180cc7253744ba31d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5f5896542231473180cc7253744ba31d 2024-11-15T17:41:57,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692577915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:57,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5f5896542231473180cc7253744ba31d, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T17:41:57,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fbf7836fe88db5084a96e3c72d0c86aa in 485ms, sequenceid=245, compaction requested=true 2024-11-15T17:41:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:41:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:57,923 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:41:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:41:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T17:41:57,923 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:57,924 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:57,924 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:57,924 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:41:57,924 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:41:57,924 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,924 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,924 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7524742cffd34dcb8ac5954f41566cf1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c51636f7406a4cc49ca78b6b0e661283, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7bb322c6ae0e45ee8af804f7467a7ae1] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=38.4 K 2024-11-15T17:41:57,924 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/bb67de5adbf14abaaf16d50cd6fd4fe2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/08f050eb35a84a6a8359273207c9de69, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eb548c91247d4421bcc94d0c2ed9777b] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=36.1 K 2024-11-15T17:41:57,925 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7524742cffd34dcb8ac5954f41566cf1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692514776 2024-11-15T17:41:57,925 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bb67de5adbf14abaaf16d50cd6fd4fe2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692514776 2024-11-15T17:41:57,925 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51636f7406a4cc49ca78b6b0e661283, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731692516103 2024-11-15T17:41:57,925 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 08f050eb35a84a6a8359273207c9de69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731692516103 2024-11-15T17:41:57,925 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb322c6ae0e45ee8af804f7467a7ae1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731692517262 2024-11-15T17:41:57,925 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting eb548c91247d4421bcc94d0c2ed9777b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731692517311 2024-11-15T17:41:57,931 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:57,932 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/eea13ba0684d4d278c450a7072037df9 is 50, key is test_row_0/B:col10/1731692517314/Put/seqid=0 2024-11-15T17:41:57,933 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:57,933 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/0135f0e8c85d42f98de93ec1702e88d2 is 50, key is test_row_0/A:col10/1731692517314/Put/seqid=0 2024-11-15T17:41:57,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742344_1520 (size=12731) 2024-11-15T17:41:57,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T17:41:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,937 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:41:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742345_1521 (size=12731) 2024-11-15T17:41:57,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/3cc2720613074be6a9cd212bdef8c89b is 50, key is test_row_0/A:col10/1731692517579/Put/seqid=0 2024-11-15T17:41:57,950 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/0135f0e8c85d42f98de93ec1702e88d2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/0135f0e8c85d42f98de93ec1702e88d2 2024-11-15T17:41:57,956 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into 0135f0e8c85d42f98de93ec1702e88d2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:57,956 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,956 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=13, startTime=1731692517923; duration=0sec 2024-11-15T17:41:57,956 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:41:57,956 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:41:57,956 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:41:57,957 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:41:57,957 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:41:57,957 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:57,957 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/2ea9029244214f5a8cd1ae979d018182, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/0a30ca983f514687a0ff08330d1a65cb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5f5896542231473180cc7253744ba31d] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=36.1 K 2024-11-15T17:41:57,957 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea9029244214f5a8cd1ae979d018182, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731692514776 2024-11-15T17:41:57,958 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a30ca983f514687a0ff08330d1a65cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731692516103 2024-11-15T17:41:57,958 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f5896542231473180cc7253744ba31d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731692517311 2024-11-15T17:41:57,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742346_1522 (size=12301) 2024-11-15T17:41:57,968 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:41:57,968 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/92c5f31ed1be4704a50f0728cc5f8413 is 50, key is test_row_0/C:col10/1731692517314/Put/seqid=0 2024-11-15T17:41:57,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742347_1523 (size=12731) 2024-11-15T17:41:57,976 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/92c5f31ed1be4704a50f0728cc5f8413 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/92c5f31ed1be4704a50f0728cc5f8413 2024-11-15T17:41:57,980 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into 92c5f31ed1be4704a50f0728cc5f8413(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:57,980 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:57,980 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=13, startTime=1731692517923; duration=0sec 2024-11-15T17:41:57,980 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:57,980 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:41:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:58,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692578032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692578033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692578034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692578034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:41:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692578145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692578145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692578145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692578156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692578225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,339 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/eea13ba0684d4d278c450a7072037df9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eea13ba0684d4d278c450a7072037df9 2024-11-15T17:41:58,342 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into eea13ba0684d4d278c450a7072037df9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:41:58,342 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:58,342 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=13, startTime=1731692517923; duration=0sec 2024-11-15T17:41:58,342 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:41:58,342 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:41:58,359 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/3cc2720613074be6a9cd212bdef8c89b 2024-11-15T17:41:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692578356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692578356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692578356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692578365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/8e489f4ee41543b49d13774fe853c208 is 50, key is test_row_0/B:col10/1731692517579/Put/seqid=0 2024-11-15T17:41:58,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742348_1524 (size=12301) 2024-11-15T17:41:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:41:58,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692578665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692578675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692578675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692578675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692578735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:58,780 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/8e489f4ee41543b49d13774fe853c208 2024-11-15T17:41:58,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/d557027f358d454f9c659d74843b18c6 is 50, key is test_row_0/C:col10/1731692517579/Put/seqid=0 2024-11-15T17:41:58,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742349_1525 (size=12301) 2024-11-15T17:41:59,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692579175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:59,189 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/d557027f358d454f9c659d74843b18c6 2024-11-15T17:41:59,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692579185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:59,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692579185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/3cc2720613074be6a9cd212bdef8c89b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/3cc2720613074be6a9cd212bdef8c89b 2024-11-15T17:41:59,197 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/3cc2720613074be6a9cd212bdef8c89b, entries=150, sequenceid=269, filesize=12.0 K 2024-11-15T17:41:59,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/8e489f4ee41543b49d13774fe853c208 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/8e489f4ee41543b49d13774fe853c208 2024-11-15T17:41:59,201 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/8e489f4ee41543b49d13774fe853c208, entries=150, sequenceid=269, filesize=12.0 K 2024-11-15T17:41:59,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/d557027f358d454f9c659d74843b18c6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d557027f358d454f9c659d74843b18c6 2024-11-15T17:41:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692579196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:59,205 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d557027f358d454f9c659d74843b18c6, entries=150, sequenceid=269, filesize=12.0 K 2024-11-15T17:41:59,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fbf7836fe88db5084a96e3c72d0c86aa in 1269ms, sequenceid=269, compaction requested=false 2024-11-15T17:41:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:41:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-15T17:41:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-15T17:41:59,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T17:41:59,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7270 sec 2024-11-15T17:41:59,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7360 sec 2024-11-15T17:41:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T17:41:59,626 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:41:59,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:41:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-15T17:41:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:41:59,628 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:41:59,628 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:41:59,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:41:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:41:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:41:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:41:59,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:41:59,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/96e78abab5cc4a49b15825814da29d9d is 50, key is test_row_0/A:col10/1731692518032/Put/seqid=0 2024-11-15T17:41:59,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742350_1526 (size=14741) 2024-11-15T17:41:59,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/96e78abab5cc4a49b15825814da29d9d 2024-11-15T17:41:59,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/b3f0e81bfda94e90b4d045d279f4b733 is 50, key is test_row_0/B:col10/1731692518032/Put/seqid=0 2024-11-15T17:41:59,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742351_1527 (size=12301) 2024-11-15T17:41:59,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/b3f0e81bfda94e90b4d045d279f4b733 2024-11-15T17:41:59,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:41:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:59,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:59,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/374f9e2fd3be40059e1d0a5133f63260 is 50, key is test_row_0/C:col10/1731692518032/Put/seqid=0 2024-11-15T17:41:59,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742352_1528 (size=12301) 2024-11-15T17:41:59,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692579873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:41:59,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:41:59,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:59,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:41:59,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:41:59,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:41:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:41:59,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:41:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692579985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:41:59,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:41:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/374f9e2fd3be40059e1d0a5133f63260 2024-11-15T17:42:00,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/96e78abab5cc4a49b15825814da29d9d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/96e78abab5cc4a49b15825814da29d9d 2024-11-15T17:42:00,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692580195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692580195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,202 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/96e78abab5cc4a49b15825814da29d9d, entries=200, sequenceid=285, filesize=14.4 K 2024-11-15T17:42:00,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/b3f0e81bfda94e90b4d045d279f4b733 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b3f0e81bfda94e90b4d045d279f4b733 2024-11-15T17:42:00,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692580206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692580206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,212 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,213 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b3f0e81bfda94e90b4d045d279f4b733, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T17:42:00,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/374f9e2fd3be40059e1d0a5133f63260 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/374f9e2fd3be40059e1d0a5133f63260 2024-11-15T17:42:00,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/374f9e2fd3be40059e1d0a5133f63260, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T17:42:00,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fbf7836fe88db5084a96e3c72d0c86aa in 469ms, sequenceid=285, compaction requested=true 2024-11-15T17:42:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:00,220 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:00,220 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:00,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:00,221 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:00,221 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:42:00,222 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,222 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eea13ba0684d4d278c450a7072037df9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/8e489f4ee41543b49d13774fe853c208, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b3f0e81bfda94e90b4d045d279f4b733] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=36.5 K 2024-11-15T17:42:00,222 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting eea13ba0684d4d278c450a7072037df9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731692517311 2024-11-15T17:42:00,222 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e489f4ee41543b49d13774fe853c208, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731692517558 2024-11-15T17:42:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:00,223 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f0e81bfda94e90b4d045d279f4b733, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731692518032 2024-11-15T17:42:00,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:00,223 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:00,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T17:42:00,223 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:42:00,223 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,223 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/0135f0e8c85d42f98de93ec1702e88d2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/3cc2720613074be6a9cd212bdef8c89b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/96e78abab5cc4a49b15825814da29d9d] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=38.8 K 2024-11-15T17:42:00,223 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0135f0e8c85d42f98de93ec1702e88d2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731692517311 2024-11-15T17:42:00,224 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc2720613074be6a9cd212bdef8c89b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731692517558 2024-11-15T17:42:00,224 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e78abab5cc4a49b15825814da29d9d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731692518032 2024-11-15T17:42:00,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a432956799fd4a2292fa096110b756ee is 50, key is test_row_0/A:col10/1731692519862/Put/seqid=0 2024-11-15T17:42:00,230 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:00,230 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/096c4a0669474a1bbe4aa1ab767ee3c4 is 50, key is test_row_0/A:col10/1731692518032/Put/seqid=0 2024-11-15T17:42:00,230 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:00,231 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/0ffa3395d9f94521b9851a8f6ebbe6b9 is 50, key is test_row_0/B:col10/1731692518032/Put/seqid=0 2024-11-15T17:42:00,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742353_1529 (size=14741) 2024-11-15T17:42:00,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a432956799fd4a2292fa096110b756ee 2024-11-15T17:42:00,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742354_1530 (size=12983) 2024-11-15T17:42:00,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742355_1531 (size=12983) 2024-11-15T17:42:00,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:00,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c0dc6c6249654622bf987da969d9f223 is 50, key is test_row_0/B:col10/1731692519862/Put/seqid=0 2024-11-15T17:42:00,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742356_1532 (size=12301) 2024-11-15T17:42:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:42:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692580292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692580405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692580505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692580625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:00,639 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/0ffa3395d9f94521b9851a8f6ebbe6b9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/0ffa3395d9f94521b9851a8f6ebbe6b9 2024-11-15T17:42:00,640 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/096c4a0669474a1bbe4aa1ab767ee3c4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/096c4a0669474a1bbe4aa1ab767ee3c4 2024-11-15T17:42:00,643 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into 0ffa3395d9f94521b9851a8f6ebbe6b9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:00,643 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:00,643 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=13, startTime=1731692520220; duration=0sec 2024-11-15T17:42:00,643 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:00,643 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:42:00,643 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:00,644 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into 096c4a0669474a1bbe4aa1ab767ee3c4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:00,644 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:00,644 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=13, startTime=1731692520220; duration=0sec 2024-11-15T17:42:00,644 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:00,644 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:42:00,644 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:00,644 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:42:00,644 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,644 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/92c5f31ed1be4704a50f0728cc5f8413, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d557027f358d454f9c659d74843b18c6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/374f9e2fd3be40059e1d0a5133f63260] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=36.5 K 2024-11-15T17:42:00,644 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c5f31ed1be4704a50f0728cc5f8413, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731692517311 2024-11-15T17:42:00,645 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting d557027f358d454f9c659d74843b18c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731692517558 2024-11-15T17:42:00,645 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 374f9e2fd3be40059e1d0a5133f63260, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731692518032 2024-11-15T17:42:00,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c0dc6c6249654622bf987da969d9f223 2024-11-15T17:42:00,650 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:00,650 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/bcfd4528f4af424795164db73b165d22 is 50, key is test_row_0/C:col10/1731692518032/Put/seqid=0 2024-11-15T17:42:00,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/1bde0bf313ef486cac17feafbac8d405 is 50, key is test_row_0/C:col10/1731692519862/Put/seqid=0 2024-11-15T17:42:00,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742357_1533 (size=12983) 2024-11-15T17:42:00,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742358_1534 (size=12301) 2024-11-15T17:42:00,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:42:00,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:00,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692580935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:01,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:01,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:01,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:01,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:01,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692581016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:01,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/1bde0bf313ef486cac17feafbac8d405 2024-11-15T17:42:01,064 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/bcfd4528f4af424795164db73b165d22 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bcfd4528f4af424795164db73b165d22 2024-11-15T17:42:01,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a432956799fd4a2292fa096110b756ee as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a432956799fd4a2292fa096110b756ee 2024-11-15T17:42:01,068 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into bcfd4528f4af424795164db73b165d22(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:01,068 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:01,068 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=13, startTime=1731692520220; duration=0sec 2024-11-15T17:42:01,068 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:01,069 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:42:01,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a432956799fd4a2292fa096110b756ee, entries=200, sequenceid=309, filesize=14.4 K 2024-11-15T17:42:01,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c0dc6c6249654622bf987da969d9f223 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0dc6c6249654622bf987da969d9f223 2024-11-15T17:42:01,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0dc6c6249654622bf987da969d9f223, entries=150, sequenceid=309, filesize=12.0 K 2024-11-15T17:42:01,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/1bde0bf313ef486cac17feafbac8d405 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/1bde0bf313ef486cac17feafbac8d405 2024-11-15T17:42:01,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/1bde0bf313ef486cac17feafbac8d405, entries=150, sequenceid=309, filesize=12.0 K 2024-11-15T17:42:01,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for fbf7836fe88db5084a96e3c72d0c86aa in 853ms, sequenceid=309, compaction requested=false 2024-11-15T17:42:01,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:01,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T17:42:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:01,154 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:42:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b is 50, key is test_row_0/A:col10/1731692520266/Put/seqid=0 2024-11-15T17:42:01,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742359_1535 (size=12301) 2024-11-15T17:42:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:01,565 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b 2024-11-15T17:42:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692581553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:01,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/587b38c007d84327848f1a3b9cbed525 is 50, key is test_row_0/B:col10/1731692520266/Put/seqid=0 2024-11-15T17:42:01,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742360_1536 (size=12301) 2024-11-15T17:42:01,573 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/587b38c007d84327848f1a3b9cbed525 2024-11-15T17:42:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/fd04ca0038474285a2d683470a4f2c40 is 50, key is test_row_0/C:col10/1731692520266/Put/seqid=0 2024-11-15T17:42:01,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742361_1537 (size=12301) 2024-11-15T17:42:01,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692581676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:42:01,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692581895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:01,984 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/fd04ca0038474285a2d683470a4f2c40 2024-11-15T17:42:01,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b 2024-11-15T17:42:01,990 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T17:42:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/587b38c007d84327848f1a3b9cbed525 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/587b38c007d84327848f1a3b9cbed525 2024-11-15T17:42:01,993 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/587b38c007d84327848f1a3b9cbed525, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T17:42:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/fd04ca0038474285a2d683470a4f2c40 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/fd04ca0038474285a2d683470a4f2c40 2024-11-15T17:42:01,995 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/fd04ca0038474285a2d683470a4f2c40, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T17:42:01,996 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fbf7836fe88db5084a96e3c72d0c86aa in 842ms, sequenceid=324, compaction requested=true 2024-11-15T17:42:01,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:01,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:01,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-15T17:42:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-15T17:42:02,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T17:42:02,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3710 sec 2024-11-15T17:42:02,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.3750 sec 2024-11-15T17:42:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:02,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:02,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:42:02,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a604b037210f454ba07b3bbd5f38551c is 50, key is test_row_0/A:col10/1731692522040/Put/seqid=0 2024-11-15T17:42:02,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742362_1538 (size=14741) 2024-11-15T17:42:02,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692582091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692582205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692582205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42164 deadline: 1731692582225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1731692582225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,229 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42110 deadline: 1731692582226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,230 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_2' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,230 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692582415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a604b037210f454ba07b3bbd5f38551c 2024-11-15T17:42:02,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/15302b42ed9e41a2ae2afa000404821a is 50, key is test_row_0/B:col10/1731692522040/Put/seqid=0 2024-11-15T17:42:02,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742363_1539 (size=12301) 2024-11-15T17:42:02,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/15302b42ed9e41a2ae2afa000404821a 2024-11-15T17:42:02,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/df5d9a7adc2642d5ae0263c20d9d4718 is 50, key is test_row_0/C:col10/1731692522040/Put/seqid=0 2024-11-15T17:42:02,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742364_1540 (size=12301) 2024-11-15T17:42:02,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692582715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692582725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:02,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/df5d9a7adc2642d5ae0263c20d9d4718 2024-11-15T17:42:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/a604b037210f454ba07b3bbd5f38551c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a604b037210f454ba07b3bbd5f38551c 2024-11-15T17:42:02,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a604b037210f454ba07b3bbd5f38551c, entries=200, sequenceid=349, filesize=14.4 K 2024-11-15T17:42:02,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/15302b42ed9e41a2ae2afa000404821a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/15302b42ed9e41a2ae2afa000404821a 2024-11-15T17:42:02,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/15302b42ed9e41a2ae2afa000404821a, entries=150, sequenceid=349, filesize=12.0 K 2024-11-15T17:42:02,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/df5d9a7adc2642d5ae0263c20d9d4718 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/df5d9a7adc2642d5ae0263c20d9d4718 2024-11-15T17:42:02,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/df5d9a7adc2642d5ae0263c20d9d4718, entries=150, sequenceid=349, filesize=12.0 K 2024-11-15T17:42:02,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fbf7836fe88db5084a96e3c72d0c86aa in 873ms, sequenceid=349, compaction requested=true 2024-11-15T17:42:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:02,915 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:42:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:02,915 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:42:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:02,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:02,915 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:42:02,915 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:42:02,915 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:42:02,915 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:42:02,916 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:02,916 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:02,916 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/096c4a0669474a1bbe4aa1ab767ee3c4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a432956799fd4a2292fa096110b756ee, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a604b037210f454ba07b3bbd5f38551c] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=53.5 K 2024-11-15T17:42:02,916 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/0ffa3395d9f94521b9851a8f6ebbe6b9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0dc6c6249654622bf987da969d9f223, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/587b38c007d84327848f1a3b9cbed525, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/15302b42ed9e41a2ae2afa000404821a] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=48.7 K 2024-11-15T17:42:02,916 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ffa3395d9f94521b9851a8f6ebbe6b9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731692518032 2024-11-15T17:42:02,916 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 096c4a0669474a1bbe4aa1ab767ee3c4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731692518032 2024-11-15T17:42:02,916 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting c0dc6c6249654622bf987da969d9f223, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731692519862 2024-11-15T17:42:02,916 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting a432956799fd4a2292fa096110b756ee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731692519845 2024-11-15T17:42:02,916 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting b69dbf49a5ca4d9c80c8b9702c0a8f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731692520266 2024-11-15T17:42:02,917 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 587b38c007d84327848f1a3b9cbed525, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731692520266 2024-11-15T17:42:02,917 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting a604b037210f454ba07b3bbd5f38551c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731692521530 2024-11-15T17:42:02,917 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 15302b42ed9e41a2ae2afa000404821a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731692521530 2024-11-15T17:42:02,923 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:02,924 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/e88d62d58af8448d8dfee6810724ee70 is 50, key is test_row_0/B:col10/1731692522040/Put/seqid=0 2024-11-15T17:42:02,924 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:02,924 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/b45bacbe4c3845838a57193fed6bdec1 is 50, key is test_row_0/A:col10/1731692522040/Put/seqid=0 2024-11-15T17:42:02,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742366_1542 (size=13119) 2024-11-15T17:42:02,934 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/e88d62d58af8448d8dfee6810724ee70 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/e88d62d58af8448d8dfee6810724ee70 2024-11-15T17:42:02,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742365_1541 (size=13119) 2024-11-15T17:42:02,937 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into e88d62d58af8448d8dfee6810724ee70(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:02,937 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:02,937 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=12, startTime=1731692522915; duration=0sec 2024-11-15T17:42:02,937 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:02,937 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:42:02,937 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T17:42:02,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T17:42:02,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:42:02,938 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:02,938 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bcfd4528f4af424795164db73b165d22, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/1bde0bf313ef486cac17feafbac8d405, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/fd04ca0038474285a2d683470a4f2c40, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/df5d9a7adc2642d5ae0263c20d9d4718] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=48.7 K 2024-11-15T17:42:02,938 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bcfd4528f4af424795164db73b165d22, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731692518032 2024-11-15T17:42:02,939 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bde0bf313ef486cac17feafbac8d405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731692519862 2024-11-15T17:42:02,939 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting fd04ca0038474285a2d683470a4f2c40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731692520266 2024-11-15T17:42:02,939 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting df5d9a7adc2642d5ae0263c20d9d4718, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731692521530 2024-11-15T17:42:02,944 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:02,944 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/3b679e895dfa481d9159bce957d2ca78 is 50, key is test_row_0/C:col10/1731692522040/Put/seqid=0 2024-11-15T17:42:02,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742367_1543 (size=13119) 2024-11-15T17:42:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:03,239 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:03,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T17:42:03,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/6364a150672c457a8c3debdff674351a is 50, key is test_row_0/A:col10/1731692522065/Put/seqid=0 2024-11-15T17:42:03,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742368_1544 (size=14741) 2024-11-15T17:42:03,336 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/b45bacbe4c3845838a57193fed6bdec1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b45bacbe4c3845838a57193fed6bdec1 2024-11-15T17:42:03,339 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into b45bacbe4c3845838a57193fed6bdec1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:03,339 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:03,339 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=12, startTime=1731692522915; duration=0sec 2024-11-15T17:42:03,339 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:03,339 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:42:03,352 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/3b679e895dfa481d9159bce957d2ca78 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/3b679e895dfa481d9159bce957d2ca78 2024-11-15T17:42:03,355 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into 3b679e895dfa481d9159bce957d2ca78(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:03,355 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:03,355 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=12, startTime=1731692522915; duration=0sec 2024-11-15T17:42:03,356 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:03,356 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:42:03,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692583386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:03,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692583495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:03,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/6364a150672c457a8c3debdff674351a 2024-11-15T17:42:03,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/a7c3a374829444948d2468d1e792d744 is 50, key is test_row_0/B:col10/1731692522065/Put/seqid=0 2024-11-15T17:42:03,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742369_1545 (size=12301) 2024-11-15T17:42:03,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692583706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:03,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:03,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692583735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:03,740 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T17:42:03,796 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:03,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:42:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-15T17:42:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:42:03,797 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:42:03,798 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:42:03,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:42:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:42:03,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:03,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692584015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:04,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/a7c3a374829444948d2468d1e792d744 2024-11-15T17:42:04,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/5e2ca7f2a35e460fbd132dfee463455e is 50, key is test_row_0/C:col10/1731692522065/Put/seqid=0 2024-11-15T17:42:04,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742370_1546 (size=12301) 2024-11-15T17:42:04,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:04,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:42:04,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:04,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:04,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:42:04,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/5e2ca7f2a35e460fbd132dfee463455e 2024-11-15T17:42:04,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/6364a150672c457a8c3debdff674351a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6364a150672c457a8c3debdff674351a 2024-11-15T17:42:04,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6364a150672c457a8c3debdff674351a, entries=200, sequenceid=362, filesize=14.4 K 2024-11-15T17:42:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/a7c3a374829444948d2468d1e792d744 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a7c3a374829444948d2468d1e792d744 2024-11-15T17:42:04,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a7c3a374829444948d2468d1e792d744, entries=150, sequenceid=362, filesize=12.0 K 2024-11-15T17:42:04,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/5e2ca7f2a35e460fbd132dfee463455e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5e2ca7f2a35e460fbd132dfee463455e 2024-11-15T17:42:04,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5e2ca7f2a35e460fbd132dfee463455e, entries=150, sequenceid=362, filesize=12.0 K 2024-11-15T17:42:04,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for fbf7836fe88db5084a96e3c72d0c86aa in 1236ms, sequenceid=362, compaction requested=false 2024-11-15T17:42:04,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:04,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:04,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:42:04,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/2c572c36d5554645b51edc44a4e659e0 is 50, key is test_row_0/A:col10/1731692523350/Put/seqid=0 2024-11-15T17:42:04,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742371_1547 (size=14741) 2024-11-15T17:42:04,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692584563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:04,587 INFO [Thread-2100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2100. 2024-11-15T17:42:04,587 DEBUG [Thread-2100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:04,587 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,587 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:04,590 INFO [Thread-2106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2106. 2024-11-15T17:42:04,590 DEBUG [Thread-2106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:04,590 DEBUG [Thread-2106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,590 DEBUG [Thread-2106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:04,592 INFO [Thread-2102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2102. 2024-11-15T17:42:04,592 DEBUG [Thread-2102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:04,592 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,592 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:04,594 INFO [Thread-2104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2104. 2024-11-15T17:42:04,594 DEBUG [Thread-2104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:04,594 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,594 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:04,598 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-11-15T17:42:04,598 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:04,598 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,598 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:04,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:04,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692584676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:04,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:04,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:04,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:04,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:04,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:04,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:04,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:04,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692584886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:04,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/2c572c36d5554645b51edc44a4e659e0 2024-11-15T17:42:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:42:04,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/514f2899524e426587d374b5007e71df is 50, key is test_row_0/B:col10/1731692523350/Put/seqid=0 2024-11-15T17:42:04,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742372_1548 (size=12301) 2024-11-15T17:42:05,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:05,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:05,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692585196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:05,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:05,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:05,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/514f2899524e426587d374b5007e71df 2024-11-15T17:42:05,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/a2121e2a507b433ca11fa45a32879986 is 50, key is test_row_0/C:col10/1731692523350/Put/seqid=0 2024-11-15T17:42:05,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742373_1549 (size=12301) 2024-11-15T17:42:05,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:05,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42158 deadline: 1731692585705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:05,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42170 deadline: 1731692585746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:05,747 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4e3b17656000,45187,1731692383005 for 'test_row_1' in fbf7836fe88db5084a96e3c72d0c86aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T17:42:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 , the old value is region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., hostname=4e3b17656000,45187,1731692383005, seqNum=2 because the exception is null or not the one we care about 2024-11-15T17:42:05,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/a2121e2a507b433ca11fa45a32879986 2024-11-15T17:42:05,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/2c572c36d5554645b51edc44a4e659e0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/2c572c36d5554645b51edc44a4e659e0 2024-11-15T17:42:05,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/2c572c36d5554645b51edc44a4e659e0, entries=200, sequenceid=389, filesize=14.4 K 2024-11-15T17:42:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/514f2899524e426587d374b5007e71df as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/514f2899524e426587d374b5007e71df 2024-11-15T17:42:05,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/514f2899524e426587d374b5007e71df, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T17:42:05,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/a2121e2a507b433ca11fa45a32879986 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/a2121e2a507b433ca11fa45a32879986 2024-11-15T17:42:05,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/a2121e2a507b433ca11fa45a32879986, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T17:42:05,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for fbf7836fe88db5084a96e3c72d0c86aa in 1261ms, sequenceid=389, compaction requested=true 2024-11-15T17:42:05,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:05,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:05,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:05,787 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:05,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:05,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:05,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fbf7836fe88db5084a96e3c72d0c86aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:05,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:05,787 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:05,788 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:05,788 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:05,788 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/A is initiating minor compaction (all files) 2024-11-15T17:42:05,788 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/B is initiating minor compaction (all files) 2024-11-15T17:42:05,788 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/A in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,788 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/B in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,788 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/e88d62d58af8448d8dfee6810724ee70, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a7c3a374829444948d2468d1e792d744, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/514f2899524e426587d374b5007e71df] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=36.8 K 2024-11-15T17:42:05,788 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b45bacbe4c3845838a57193fed6bdec1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6364a150672c457a8c3debdff674351a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/2c572c36d5554645b51edc44a4e659e0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=41.6 K 2024-11-15T17:42:05,788 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting b45bacbe4c3845838a57193fed6bdec1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731692521530 2024-11-15T17:42:05,788 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting e88d62d58af8448d8dfee6810724ee70, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731692521530 2024-11-15T17:42:05,789 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6364a150672c457a8c3debdff674351a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731692522065 2024-11-15T17:42:05,789 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c3a374829444948d2468d1e792d744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731692522065 2024-11-15T17:42:05,789 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c572c36d5554645b51edc44a4e659e0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731692523350 2024-11-15T17:42:05,789 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 514f2899524e426587d374b5007e71df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731692523350 2024-11-15T17:42:05,794 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#B#compaction#465 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:05,794 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:05,794 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/32f0c95d9c684c8183607d936d394bf9 is 50, key is test_row_0/B:col10/1731692523350/Put/seqid=0 2024-11-15T17:42:05,794 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/dd396cd5c3d240c881f22d2d1a08864d is 50, key is test_row_0/A:col10/1731692523350/Put/seqid=0 2024-11-15T17:42:05,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742374_1550 (size=13221) 2024-11-15T17:42:05,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742375_1551 (size=13221) 2024-11-15T17:42:05,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T17:42:05,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:05,807 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:42:05,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/54d30329ada346cdb85571969d30cd73 is 50, key is test_row_0/A:col10/1731692524562/Put/seqid=0 2024-11-15T17:42:05,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742376_1552 (size=12301) 2024-11-15T17:42:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:42:06,207 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/dd396cd5c3d240c881f22d2d1a08864d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/dd396cd5c3d240c881f22d2d1a08864d 2024-11-15T17:42:06,209 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/32f0c95d9c684c8183607d936d394bf9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/32f0c95d9c684c8183607d936d394bf9 2024-11-15T17:42:06,213 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/54d30329ada346cdb85571969d30cd73 2024-11-15T17:42:06,214 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/A of fbf7836fe88db5084a96e3c72d0c86aa into dd396cd5c3d240c881f22d2d1a08864d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:06,214 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/B of fbf7836fe88db5084a96e3c72d0c86aa into 32f0c95d9c684c8183607d936d394bf9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:06,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:06,214 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:06,214 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/A, priority=13, startTime=1731692525787; duration=0sec 2024-11-15T17:42:06,214 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/B, priority=13, startTime=1731692525787; duration=0sec 2024-11-15T17:42:06,214 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:06,214 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:B 2024-11-15T17:42:06,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:06,214 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:06,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:A 2024-11-15T17:42:06,215 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:06,215 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): fbf7836fe88db5084a96e3c72d0c86aa/C is initiating minor compaction (all files) 2024-11-15T17:42:06,215 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fbf7836fe88db5084a96e3c72d0c86aa/C in TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:06,215 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/3b679e895dfa481d9159bce957d2ca78, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5e2ca7f2a35e460fbd132dfee463455e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/a2121e2a507b433ca11fa45a32879986] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp, totalSize=36.8 K 2024-11-15T17:42:06,215 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b679e895dfa481d9159bce957d2ca78, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731692521530 2024-11-15T17:42:06,216 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2ca7f2a35e460fbd132dfee463455e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731692522065 2024-11-15T17:42:06,216 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting a2121e2a507b433ca11fa45a32879986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731692523350 2024-11-15T17:42:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/b6d0b405d1094999958bb8dd4954017b is 50, key is test_row_0/B:col10/1731692524562/Put/seqid=0 2024-11-15T17:42:06,222 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fbf7836fe88db5084a96e3c72d0c86aa#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:06,222 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/b2666620a10a4ea0ae70b3fef8d39069 is 50, key is test_row_0/C:col10/1731692523350/Put/seqid=0 2024-11-15T17:42:06,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742377_1553 (size=12301) 2024-11-15T17:42:06,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742378_1554 (size=13221) 2024-11-15T17:42:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. as already flushing 2024-11-15T17:42:06,257 INFO [Thread-2091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2091. 2024-11-15T17:42:06,257 DEBUG [Thread-2091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:06,257 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,257 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:06,268 INFO [Thread-2087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2087. 2024-11-15T17:42:06,268 INFO [Thread-2089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2089. 2024-11-15T17:42:06,268 DEBUG [Thread-2087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:06,268 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,268 DEBUG [Thread-2089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:06,268 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,268 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,268 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:06,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:06,625 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/b6d0b405d1094999958bb8dd4954017b 2024-11-15T17:42:06,633 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/b2666620a10a4ea0ae70b3fef8d39069 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b2666620a10a4ea0ae70b3fef8d39069 2024-11-15T17:42:06,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/d56995966ebf4d68b14933f7dcbe1abe is 50, key is test_row_0/C:col10/1731692524562/Put/seqid=0 2024-11-15T17:42:06,638 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fbf7836fe88db5084a96e3c72d0c86aa/C of fbf7836fe88db5084a96e3c72d0c86aa into b2666620a10a4ea0ae70b3fef8d39069(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:06,638 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:06,638 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., storeName=fbf7836fe88db5084a96e3c72d0c86aa/C, priority=13, startTime=1731692525787; duration=0sec 2024-11-15T17:42:06,638 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:06,638 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fbf7836fe88db5084a96e3c72d0c86aa:C 2024-11-15T17:42:06,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742379_1555 (size=12301) 2024-11-15T17:42:06,720 INFO [Thread-2093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2093. 2024-11-15T17:42:06,720 DEBUG [Thread-2093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:06,720 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,721 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:06,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:07,040 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/d56995966ebf4d68b14933f7dcbe1abe 2024-11-15T17:42:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/54d30329ada346cdb85571969d30cd73 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/54d30329ada346cdb85571969d30cd73 2024-11-15T17:42:07,052 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/54d30329ada346cdb85571969d30cd73, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:42:07,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/b6d0b405d1094999958bb8dd4954017b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b6d0b405d1094999958bb8dd4954017b 2024-11-15T17:42:07,057 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b6d0b405d1094999958bb8dd4954017b, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:42:07,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/d56995966ebf4d68b14933f7dcbe1abe as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d56995966ebf4d68b14933f7dcbe1abe 2024-11-15T17:42:07,063 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d56995966ebf4d68b14933f7dcbe1abe, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T17:42:07,064 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=26.84 KB/27480 for fbf7836fe88db5084a96e3c72d0c86aa in 1257ms, sequenceid=400, compaction requested=false 2024-11-15T17:42:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for fbf7836fe88db5084a96e3c72d0c86aa: 2024-11-15T17:42:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:07,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-15T17:42:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-15T17:42:07,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T17:42:07,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2670 sec 2024-11-15T17:42:07,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 3.2730 sec 2024-11-15T17:42:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T17:42:07,968 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:09,768 INFO [Thread-2095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2095. 2024-11-15T17:42:09,769 DEBUG [Thread-2095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:09,769 DEBUG [Thread-2095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:09,769 DEBUG [Thread-2095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:09,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:09,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:42:09,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-15T17:42:09,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-15T17:42:09,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-15T17:42:09,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 197 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1573 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4710 rows 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1577 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4730 rows 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1579 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4730 rows 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1575 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4724 rows 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1563 2024-11-15T17:42:09,770 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4686 rows 2024-11-15T17:42:09,770 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:42:09,770 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:42:09,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:09,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:09,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:09,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:42:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:42:09,778 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692529778"}]},"ts":"1731692529778"} 2024-11-15T17:42:09,780 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:42:09,780 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:42:09,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:42:09,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, UNASSIGN}] 2024-11-15T17:42:09,784 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, UNASSIGN 2024-11-15T17:42:09,785 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=fbf7836fe88db5084a96e3c72d0c86aa, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:42:09,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, UNASSIGN because future has completed 2024-11-15T17:42:09,786 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:42:09,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:42:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:42:09,941 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:09,941 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:42:09,941 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing fbf7836fe88db5084a96e3c72d0c86aa, disabling compactions & flushes 2024-11-15T17:42:09,941 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:09,941 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:09,941 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. after waiting 0 ms 2024-11-15T17:42:09,941 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:09,942 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing fbf7836fe88db5084a96e3c72d0c86aa 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T17:42:09,950 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c2adf5dc2d144dc0802c98cb2d29db30 is 50, key is test_row_1/A:col10/1731692529766/Put/seqid=0 2024-11-15T17:42:09,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742380_1556 (size=9857) 2024-11-15T17:42:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:42:10,355 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c2adf5dc2d144dc0802c98cb2d29db30 2024-11-15T17:42:10,368 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c14a87feef7c4562907841aaa075310a is 50, key is test_row_1/B:col10/1731692529766/Put/seqid=0 2024-11-15T17:42:10,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742381_1557 (size=9857) 2024-11-15T17:42:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:42:10,773 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c14a87feef7c4562907841aaa075310a 2024-11-15T17:42:10,787 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/4b8d132114024d448e4982538861e40d is 50, key is test_row_1/C:col10/1731692529766/Put/seqid=0 2024-11-15T17:42:10,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742382_1558 (size=9857) 2024-11-15T17:42:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:42:11,192 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/4b8d132114024d448e4982538861e40d 2024-11-15T17:42:11,202 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/A/c2adf5dc2d144dc0802c98cb2d29db30 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c2adf5dc2d144dc0802c98cb2d29db30 2024-11-15T17:42:11,206 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c2adf5dc2d144dc0802c98cb2d29db30, entries=100, sequenceid=411, filesize=9.6 K 2024-11-15T17:42:11,206 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/B/c14a87feef7c4562907841aaa075310a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c14a87feef7c4562907841aaa075310a 2024-11-15T17:42:11,210 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c14a87feef7c4562907841aaa075310a, entries=100, sequenceid=411, filesize=9.6 K 2024-11-15T17:42:11,210 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/.tmp/C/4b8d132114024d448e4982538861e40d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/4b8d132114024d448e4982538861e40d 2024-11-15T17:42:11,214 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/4b8d132114024d448e4982538861e40d, entries=100, sequenceid=411, filesize=9.6 K 2024-11-15T17:42:11,214 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fbf7836fe88db5084a96e3c72d0c86aa in 1273ms, sequenceid=411, compaction requested=true 2024-11-15T17:42:11,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7ef164f975c64ea48992b431dc62ce00, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/d7adf642e1fa43bf99f10b59e0099951, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/73c5dda63ceb49f9b4d3f9b694a0b382, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/234945c1dccd4275b7e076c663fd443e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/956707a75381426982b45def124ba413, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/77aafeec4e1f4d62a6e6295f3c46d60b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c58fe9f0f57b402199485cbb215d45f5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6627b321ef2a487dabbb81ae8092672d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a02e4387ff8e410b9f2670b72df10f21, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9a64d36af302403da1121df65fa7f6c2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c901f585555548bd8c75a8ce2eb0a00e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7b01b3fbbce54cd6b63e33496c9c572c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9650063d8bf242eca9c9f009bdb8000e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7524742cffd34dcb8ac5954f41566cf1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/49b7643bf3ca447ba4bc59f0906a898b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c51636f7406a4cc49ca78b6b0e661283, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7bb322c6ae0e45ee8af804f7467a7ae1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/0135f0e8c85d42f98de93ec1702e88d2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/3cc2720613074be6a9cd212bdef8c89b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/96e78abab5cc4a49b15825814da29d9d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/096c4a0669474a1bbe4aa1ab767ee3c4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a432956799fd4a2292fa096110b756ee, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a604b037210f454ba07b3bbd5f38551c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b45bacbe4c3845838a57193fed6bdec1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6364a150672c457a8c3debdff674351a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/2c572c36d5554645b51edc44a4e659e0] to archive 2024-11-15T17:42:11,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:42:11,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7ef164f975c64ea48992b431dc62ce00 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7ef164f975c64ea48992b431dc62ce00 2024-11-15T17:42:11,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/d7adf642e1fa43bf99f10b59e0099951 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/d7adf642e1fa43bf99f10b59e0099951 2024-11-15T17:42:11,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/73c5dda63ceb49f9b4d3f9b694a0b382 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/73c5dda63ceb49f9b4d3f9b694a0b382 2024-11-15T17:42:11,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/234945c1dccd4275b7e076c663fd443e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/234945c1dccd4275b7e076c663fd443e 2024-11-15T17:42:11,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/956707a75381426982b45def124ba413 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/956707a75381426982b45def124ba413 2024-11-15T17:42:11,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/77aafeec4e1f4d62a6e6295f3c46d60b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/77aafeec4e1f4d62a6e6295f3c46d60b 2024-11-15T17:42:11,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c58fe9f0f57b402199485cbb215d45f5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c58fe9f0f57b402199485cbb215d45f5 2024-11-15T17:42:11,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6627b321ef2a487dabbb81ae8092672d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6627b321ef2a487dabbb81ae8092672d 2024-11-15T17:42:11,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a02e4387ff8e410b9f2670b72df10f21 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a02e4387ff8e410b9f2670b72df10f21 2024-11-15T17:42:11,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9a64d36af302403da1121df65fa7f6c2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9a64d36af302403da1121df65fa7f6c2 2024-11-15T17:42:11,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c901f585555548bd8c75a8ce2eb0a00e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c901f585555548bd8c75a8ce2eb0a00e 2024-11-15T17:42:11,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7b01b3fbbce54cd6b63e33496c9c572c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7b01b3fbbce54cd6b63e33496c9c572c 2024-11-15T17:42:11,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9650063d8bf242eca9c9f009bdb8000e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/9650063d8bf242eca9c9f009bdb8000e 2024-11-15T17:42:11,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7524742cffd34dcb8ac5954f41566cf1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7524742cffd34dcb8ac5954f41566cf1 2024-11-15T17:42:11,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/49b7643bf3ca447ba4bc59f0906a898b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/49b7643bf3ca447ba4bc59f0906a898b 2024-11-15T17:42:11,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c51636f7406a4cc49ca78b6b0e661283 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c51636f7406a4cc49ca78b6b0e661283 2024-11-15T17:42:11,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7bb322c6ae0e45ee8af804f7467a7ae1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/7bb322c6ae0e45ee8af804f7467a7ae1 2024-11-15T17:42:11,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/0135f0e8c85d42f98de93ec1702e88d2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/0135f0e8c85d42f98de93ec1702e88d2 2024-11-15T17:42:11,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/3cc2720613074be6a9cd212bdef8c89b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/3cc2720613074be6a9cd212bdef8c89b 2024-11-15T17:42:11,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/96e78abab5cc4a49b15825814da29d9d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/96e78abab5cc4a49b15825814da29d9d 2024-11-15T17:42:11,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/096c4a0669474a1bbe4aa1ab767ee3c4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/096c4a0669474a1bbe4aa1ab767ee3c4 2024-11-15T17:42:11,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a432956799fd4a2292fa096110b756ee to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a432956799fd4a2292fa096110b756ee 2024-11-15T17:42:11,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b69dbf49a5ca4d9c80c8b9702c0a8f0b 2024-11-15T17:42:11,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a604b037210f454ba07b3bbd5f38551c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/a604b037210f454ba07b3bbd5f38551c 2024-11-15T17:42:11,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b45bacbe4c3845838a57193fed6bdec1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/b45bacbe4c3845838a57193fed6bdec1 2024-11-15T17:42:11,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6364a150672c457a8c3debdff674351a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/6364a150672c457a8c3debdff674351a 2024-11-15T17:42:11,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:42:11,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/2c572c36d5554645b51edc44a4e659e0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/2c572c36d5554645b51edc44a4e659e0 2024-11-15T17:42:11,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/9a46f9daea5240c08762e3ff23a5cde6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a9049a1134df47c1965904f0b483fcc5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/03f86364a1e5400f92ca87b0ff124fc1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/ac785128a9e1492ebdb41fdc1c3711c5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0746bb17e074820a383b13645a98575, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d975774ba2ee4bb2aa3588ae876f4f8c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/fcc3d87df37d46dda31555397b537555, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/6119f8b2a0f5497da75d77b3b07e9d1c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/49cda229dd3d49aea6b305f33e8824af, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/7bc06f927afa4647aca289e87544f845, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/73a6e86a679042c1b3b6483dc8ceece0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/5887f85457d14f9db31048efe6b49768, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d3fbc31ce3a449129e5069949b6561eb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/bb67de5adbf14abaaf16d50cd6fd4fe2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/41db1e3e24a2456e8bd7a46a43981629, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/08f050eb35a84a6a8359273207c9de69, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eea13ba0684d4d278c450a7072037df9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eb548c91247d4421bcc94d0c2ed9777b, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/8e489f4ee41543b49d13774fe853c208, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/0ffa3395d9f94521b9851a8f6ebbe6b9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b3f0e81bfda94e90b4d045d279f4b733, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0dc6c6249654622bf987da969d9f223, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/587b38c007d84327848f1a3b9cbed525, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/e88d62d58af8448d8dfee6810724ee70, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/15302b42ed9e41a2ae2afa000404821a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a7c3a374829444948d2468d1e792d744, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/514f2899524e426587d374b5007e71df] to archive 2024-11-15T17:42:11,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:42:11,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/9a46f9daea5240c08762e3ff23a5cde6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/9a46f9daea5240c08762e3ff23a5cde6 2024-11-15T17:42:11,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a9049a1134df47c1965904f0b483fcc5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a9049a1134df47c1965904f0b483fcc5 2024-11-15T17:42:11,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/03f86364a1e5400f92ca87b0ff124fc1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/03f86364a1e5400f92ca87b0ff124fc1 2024-11-15T17:42:11,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/ac785128a9e1492ebdb41fdc1c3711c5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/ac785128a9e1492ebdb41fdc1c3711c5 2024-11-15T17:42:11,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0746bb17e074820a383b13645a98575 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0746bb17e074820a383b13645a98575 2024-11-15T17:42:11,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d975774ba2ee4bb2aa3588ae876f4f8c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d975774ba2ee4bb2aa3588ae876f4f8c 2024-11-15T17:42:11,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/fcc3d87df37d46dda31555397b537555 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/fcc3d87df37d46dda31555397b537555 2024-11-15T17:42:11,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/6119f8b2a0f5497da75d77b3b07e9d1c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/6119f8b2a0f5497da75d77b3b07e9d1c 2024-11-15T17:42:11,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/49cda229dd3d49aea6b305f33e8824af to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/49cda229dd3d49aea6b305f33e8824af 2024-11-15T17:42:11,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/7bc06f927afa4647aca289e87544f845 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/7bc06f927afa4647aca289e87544f845 2024-11-15T17:42:11,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/73a6e86a679042c1b3b6483dc8ceece0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/73a6e86a679042c1b3b6483dc8ceece0 2024-11-15T17:42:11,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/5887f85457d14f9db31048efe6b49768 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/5887f85457d14f9db31048efe6b49768 2024-11-15T17:42:11,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d3fbc31ce3a449129e5069949b6561eb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/d3fbc31ce3a449129e5069949b6561eb 2024-11-15T17:42:11,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/bb67de5adbf14abaaf16d50cd6fd4fe2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/bb67de5adbf14abaaf16d50cd6fd4fe2 2024-11-15T17:42:11,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/41db1e3e24a2456e8bd7a46a43981629 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/41db1e3e24a2456e8bd7a46a43981629 2024-11-15T17:42:11,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/08f050eb35a84a6a8359273207c9de69 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/08f050eb35a84a6a8359273207c9de69 2024-11-15T17:42:11,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eea13ba0684d4d278c450a7072037df9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eea13ba0684d4d278c450a7072037df9 2024-11-15T17:42:11,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eb548c91247d4421bcc94d0c2ed9777b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/eb548c91247d4421bcc94d0c2ed9777b 2024-11-15T17:42:11,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/8e489f4ee41543b49d13774fe853c208 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/8e489f4ee41543b49d13774fe853c208 2024-11-15T17:42:11,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/0ffa3395d9f94521b9851a8f6ebbe6b9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/0ffa3395d9f94521b9851a8f6ebbe6b9 2024-11-15T17:42:11,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b3f0e81bfda94e90b4d045d279f4b733 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b3f0e81bfda94e90b4d045d279f4b733 2024-11-15T17:42:11,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0dc6c6249654622bf987da969d9f223 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c0dc6c6249654622bf987da969d9f223 2024-11-15T17:42:11,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/587b38c007d84327848f1a3b9cbed525 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/587b38c007d84327848f1a3b9cbed525 2024-11-15T17:42:11,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/e88d62d58af8448d8dfee6810724ee70 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/e88d62d58af8448d8dfee6810724ee70 2024-11-15T17:42:11,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/15302b42ed9e41a2ae2afa000404821a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/15302b42ed9e41a2ae2afa000404821a 2024-11-15T17:42:11,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a7c3a374829444948d2468d1e792d744 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/a7c3a374829444948d2468d1e792d744 2024-11-15T17:42:11,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/514f2899524e426587d374b5007e71df to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/514f2899524e426587d374b5007e71df 2024-11-15T17:42:11,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/34a20067c35a4c498e4696080a11bbaa, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/20ef2e6861e44e13a33bc851bd1ce4e4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/17ca2ebfc6054046b2f7a9e7462975a4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/58b88538278b4fe982f6f4ce6f7e7d68, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/9459a0d6fa8841309063a48a2a6d2132, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/c2fa5c057e894b33a9f171c0fe98ab59, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bbe5048741ec46dbb8317c0fa03dc2a5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/137295b3308e45c594b01ad22b4c2f4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/8a7e8c1bbdc247d7afe403dec5f55966, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bb862a4d1c034565ac8ef644c40493c7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/507ceadd168c41209cb07d087510f8de, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/733556605fdd4f5da8b69582ce7a8d29, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/87e16ed10021424796573e8e31be652e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/2ea9029244214f5a8cd1ae979d018182, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b70294b0f32a45acba0a99b8eddaeb9e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/0a30ca983f514687a0ff08330d1a65cb, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/92c5f31ed1be4704a50f0728cc5f8413, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5f5896542231473180cc7253744ba31d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d557027f358d454f9c659d74843b18c6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bcfd4528f4af424795164db73b165d22, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/374f9e2fd3be40059e1d0a5133f63260, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/1bde0bf313ef486cac17feafbac8d405, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/fd04ca0038474285a2d683470a4f2c40, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/3b679e895dfa481d9159bce957d2ca78, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/df5d9a7adc2642d5ae0263c20d9d4718, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5e2ca7f2a35e460fbd132dfee463455e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/a2121e2a507b433ca11fa45a32879986] to archive 2024-11-15T17:42:11,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:42:11,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/34a20067c35a4c498e4696080a11bbaa to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/34a20067c35a4c498e4696080a11bbaa 2024-11-15T17:42:11,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/20ef2e6861e44e13a33bc851bd1ce4e4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/20ef2e6861e44e13a33bc851bd1ce4e4 2024-11-15T17:42:11,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/17ca2ebfc6054046b2f7a9e7462975a4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/17ca2ebfc6054046b2f7a9e7462975a4 2024-11-15T17:42:11,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/58b88538278b4fe982f6f4ce6f7e7d68 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/58b88538278b4fe982f6f4ce6f7e7d68 2024-11-15T17:42:11,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/9459a0d6fa8841309063a48a2a6d2132 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/9459a0d6fa8841309063a48a2a6d2132 2024-11-15T17:42:11,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/c2fa5c057e894b33a9f171c0fe98ab59 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/c2fa5c057e894b33a9f171c0fe98ab59 2024-11-15T17:42:11,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bbe5048741ec46dbb8317c0fa03dc2a5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bbe5048741ec46dbb8317c0fa03dc2a5 2024-11-15T17:42:11,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/137295b3308e45c594b01ad22b4c2f4e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/137295b3308e45c594b01ad22b4c2f4e 2024-11-15T17:42:11,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/8a7e8c1bbdc247d7afe403dec5f55966 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/8a7e8c1bbdc247d7afe403dec5f55966 2024-11-15T17:42:11,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bb862a4d1c034565ac8ef644c40493c7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bb862a4d1c034565ac8ef644c40493c7 2024-11-15T17:42:11,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/507ceadd168c41209cb07d087510f8de to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/507ceadd168c41209cb07d087510f8de 2024-11-15T17:42:11,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/733556605fdd4f5da8b69582ce7a8d29 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/733556605fdd4f5da8b69582ce7a8d29 2024-11-15T17:42:11,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/87e16ed10021424796573e8e31be652e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/87e16ed10021424796573e8e31be652e 2024-11-15T17:42:11,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/2ea9029244214f5a8cd1ae979d018182 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/2ea9029244214f5a8cd1ae979d018182 2024-11-15T17:42:11,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b70294b0f32a45acba0a99b8eddaeb9e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b70294b0f32a45acba0a99b8eddaeb9e 2024-11-15T17:42:11,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/0a30ca983f514687a0ff08330d1a65cb to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/0a30ca983f514687a0ff08330d1a65cb 2024-11-15T17:42:11,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/92c5f31ed1be4704a50f0728cc5f8413 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/92c5f31ed1be4704a50f0728cc5f8413 2024-11-15T17:42:11,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5f5896542231473180cc7253744ba31d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5f5896542231473180cc7253744ba31d 2024-11-15T17:42:11,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d557027f358d454f9c659d74843b18c6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d557027f358d454f9c659d74843b18c6 2024-11-15T17:42:11,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bcfd4528f4af424795164db73b165d22 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/bcfd4528f4af424795164db73b165d22 2024-11-15T17:42:11,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/374f9e2fd3be40059e1d0a5133f63260 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/374f9e2fd3be40059e1d0a5133f63260 2024-11-15T17:42:11,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/1bde0bf313ef486cac17feafbac8d405 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/1bde0bf313ef486cac17feafbac8d405 2024-11-15T17:42:11,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/fd04ca0038474285a2d683470a4f2c40 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/fd04ca0038474285a2d683470a4f2c40 2024-11-15T17:42:11,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/3b679e895dfa481d9159bce957d2ca78 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/3b679e895dfa481d9159bce957d2ca78 2024-11-15T17:42:11,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/df5d9a7adc2642d5ae0263c20d9d4718 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/df5d9a7adc2642d5ae0263c20d9d4718 2024-11-15T17:42:11,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5e2ca7f2a35e460fbd132dfee463455e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/5e2ca7f2a35e460fbd132dfee463455e 2024-11-15T17:42:11,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/a2121e2a507b433ca11fa45a32879986 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/a2121e2a507b433ca11fa45a32879986 2024-11-15T17:42:11,301 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/recovered.edits/414.seqid, newMaxSeqId=414, maxSeqId=1 2024-11-15T17:42:11,302 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. 2024-11-15T17:42:11,302 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for fbf7836fe88db5084a96e3c72d0c86aa: Waiting for close lock at 1731692529941Running coprocessor pre-close hooks at 1731692529941Disabling compacts and flushes for region at 1731692529941Disabling writes for close at 1731692529941Obtaining lock to block concurrent updates at 1731692529942 (+1 ms)Preparing flush snapshotting stores in fbf7836fe88db5084a96e3c72d0c86aa at 1731692529942Finished memstore snapshotting TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731692529942Flushing stores of TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa. at 1731692529944 (+2 ms)Flushing fbf7836fe88db5084a96e3c72d0c86aa/A: creating writer at 1731692529944Flushing fbf7836fe88db5084a96e3c72d0c86aa/A: appending metadata at 1731692529950 (+6 ms)Flushing fbf7836fe88db5084a96e3c72d0c86aa/A: closing flushed file at 1731692529950Flushing fbf7836fe88db5084a96e3c72d0c86aa/B: creating writer at 1731692530365 (+415 ms)Flushing fbf7836fe88db5084a96e3c72d0c86aa/B: appending metadata at 1731692530367 (+2 ms)Flushing fbf7836fe88db5084a96e3c72d0c86aa/B: closing flushed file at 1731692530368 (+1 ms)Flushing fbf7836fe88db5084a96e3c72d0c86aa/C: creating writer at 1731692530783 (+415 ms)Flushing fbf7836fe88db5084a96e3c72d0c86aa/C: appending metadata at 1731692530787 (+4 ms)Flushing fbf7836fe88db5084a96e3c72d0c86aa/C: closing flushed file at 1731692530787Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5139eb6a: reopening flushed file at 1731692531201 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e2ef01c: reopening flushed file at 1731692531206 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d59bad7: reopening flushed file at 1731692531210 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for fbf7836fe88db5084a96e3c72d0c86aa in 1273ms, sequenceid=411, compaction requested=true at 1731692531214 (+4 ms)Writing region close event to WAL at 1731692531299 (+85 ms)Running coprocessor post-close hooks at 1731692531302 (+3 ms)Closed at 1731692531302 2024-11-15T17:42:11,304 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:11,304 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=fbf7836fe88db5084a96e3c72d0c86aa, regionState=CLOSED 2024-11-15T17:42:11,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:42:11,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-15T17:42:11,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure fbf7836fe88db5084a96e3c72d0c86aa, server=4e3b17656000,45187,1731692383005 in 1.5200 sec 2024-11-15T17:42:11,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-15T17:42:11,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fbf7836fe88db5084a96e3c72d0c86aa, UNASSIGN in 1.5250 sec 2024-11-15T17:42:11,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T17:42:11,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5290 sec 2024-11-15T17:42:11,312 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692531312"}]},"ts":"1731692531312"} 2024-11-15T17:42:11,313 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:42:11,313 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:42:11,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5410 sec 2024-11-15T17:42:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T17:42:11,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:11,928 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:42:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:11,931 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:42:11,933 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:11,936 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:11,940 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/recovered.edits] 2024-11-15T17:42:11,944 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/54d30329ada346cdb85571969d30cd73 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/54d30329ada346cdb85571969d30cd73 2024-11-15T17:42:11,946 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c2adf5dc2d144dc0802c98cb2d29db30 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/c2adf5dc2d144dc0802c98cb2d29db30 2024-11-15T17:42:11,948 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/dd396cd5c3d240c881f22d2d1a08864d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/A/dd396cd5c3d240c881f22d2d1a08864d 2024-11-15T17:42:11,952 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/32f0c95d9c684c8183607d936d394bf9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/32f0c95d9c684c8183607d936d394bf9 2024-11-15T17:42:11,954 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b6d0b405d1094999958bb8dd4954017b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/b6d0b405d1094999958bb8dd4954017b 2024-11-15T17:42:11,956 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c14a87feef7c4562907841aaa075310a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/B/c14a87feef7c4562907841aaa075310a 2024-11-15T17:42:11,960 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/4b8d132114024d448e4982538861e40d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/4b8d132114024d448e4982538861e40d 2024-11-15T17:42:11,962 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b2666620a10a4ea0ae70b3fef8d39069 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/b2666620a10a4ea0ae70b3fef8d39069 2024-11-15T17:42:11,964 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d56995966ebf4d68b14933f7dcbe1abe to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/C/d56995966ebf4d68b14933f7dcbe1abe 2024-11-15T17:42:11,968 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/recovered.edits/414.seqid to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa/recovered.edits/414.seqid 2024-11-15T17:42:11,969 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/fbf7836fe88db5084a96e3c72d0c86aa 2024-11-15T17:42:11,970 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:42:11,973 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:11,977 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:42:11,980 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:42:11,981 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:11,981 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:42:11,982 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731692531981"}]},"ts":"9223372036854775807"} 2024-11-15T17:42:11,984 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:42:11,984 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fbf7836fe88db5084a96e3c72d0c86aa, NAME => 'TestAcidGuarantees,,1731692502381.fbf7836fe88db5084a96e3c72d0c86aa.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:42:11,984 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:42:11,985 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731692531984"}]},"ts":"9223372036854775807"} 2024-11-15T17:42:11,987 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:42:11,988 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:11,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-15T17:42:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T17:42:12,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:42:12,044 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:12,053 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 237), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=9389 (was 9412) 2024-11-15T17:42:12,061 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=359, ProcessCount=11, AvailableMemoryMB=9388 2024-11-15T17:42:12,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:42:12,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:42:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:12,064 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:42:12,064 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:12,064 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-15T17:42:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T17:42:12,065 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:42:12,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742383_1559 (size=959) 2024-11-15T17:42:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T17:42:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T17:42:12,475 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e 2024-11-15T17:42:12,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742384_1560 (size=53) 2024-11-15T17:42:12,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T17:42:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T17:42:12,884 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:42:12,884 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 248b48bc55c0fdcd5620a92777fad08f, disabling compactions & flushes 2024-11-15T17:42:12,884 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:12,884 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:12,884 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. after waiting 0 ms 2024-11-15T17:42:12,884 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:12,884 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:12,884 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 248b48bc55c0fdcd5620a92777fad08f: Waiting for close lock at 1731692532884Disabling compacts and flushes for region at 1731692532884Disabling writes for close at 1731692532884Writing region close event to WAL at 1731692532884Closed at 1731692532884 2024-11-15T17:42:12,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:42:12,886 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731692532885"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692532885"}]},"ts":"1731692532885"} 2024-11-15T17:42:12,888 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:42:12,888 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:42:12,889 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692532888"}]},"ts":"1731692532888"} 2024-11-15T17:42:12,890 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T17:42:12,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, ASSIGN}] 2024-11-15T17:42:12,892 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, ASSIGN 2024-11-15T17:42:12,893 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, ASSIGN; state=OFFLINE, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=false 2024-11-15T17:42:13,044 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:42:13,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, ASSIGN because future has completed 2024-11-15T17:42:13,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:42:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T17:42:13,209 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:13,209 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:42:13,209 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,209 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:42:13,209 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,209 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,211 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,213 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 248b48bc55c0fdcd5620a92777fad08f columnFamilyName A 2024-11-15T17:42:13,213 DEBUG [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:13,214 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(327): Store=248b48bc55c0fdcd5620a92777fad08f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:42:13,214 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,216 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 248b48bc55c0fdcd5620a92777fad08f columnFamilyName B 2024-11-15T17:42:13,216 DEBUG [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:13,216 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(327): Store=248b48bc55c0fdcd5620a92777fad08f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:42:13,217 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,218 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 248b48bc55c0fdcd5620a92777fad08f columnFamilyName C 2024-11-15T17:42:13,218 DEBUG [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:13,219 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(327): Store=248b48bc55c0fdcd5620a92777fad08f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:42:13,219 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,220 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,221 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,223 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,223 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,224 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:42:13,226 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,229 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:42:13,230 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 248b48bc55c0fdcd5620a92777fad08f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64884344, jitterRate=-0.03314793109893799}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:42:13,230 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:13,231 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 248b48bc55c0fdcd5620a92777fad08f: Running coprocessor pre-open hook at 1731692533210Writing region info on filesystem at 1731692533210Initializing all the Stores at 1731692533210Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692533211 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692533211Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692533211Cleaning up temporary data from old regions at 1731692533223 (+12 ms)Running coprocessor post-open hooks at 1731692533230 (+7 ms)Region opened successfully at 1731692533231 (+1 ms) 2024-11-15T17:42:13,232 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., pid=146, masterSystemTime=1731692533201 2024-11-15T17:42:13,233 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:13,233 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:13,234 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=OPEN, openSeqNum=2, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:42:13,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:42:13,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-15T17:42:13,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 in 188 msec 2024-11-15T17:42:13,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-15T17:42:13,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, ASSIGN in 347 msec 2024-11-15T17:42:13,241 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:42:13,241 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692533241"}]},"ts":"1731692533241"} 2024-11-15T17:42:13,243 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T17:42:13,244 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:42:13,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-15T17:42:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T17:42:14,217 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:14,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:14,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:14,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:14,221 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce99180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:14,225 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db0b96d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:14,226 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:14,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:14,229 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:42:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 4e3b17656000,41205,1731692382118 2024-11-15T17:42:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a80bdb5 2024-11-15T17:42:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:42:14,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:42:14,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T17:42:14,232 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:42:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:14,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742385_1561 (size=995) 2024-11-15T17:42:14,645 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T17:42:14,645 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T17:42:14,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:42:14,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, REOPEN/MOVE}] 2024-11-15T17:42:14,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, REOPEN/MOVE 2024-11-15T17:42:14,654 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:42:14,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, REOPEN/MOVE because future has completed 2024-11-15T17:42:14,656 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:42:14,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:42:14,808 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:14,808 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:42:14,808 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 248b48bc55c0fdcd5620a92777fad08f, disabling compactions & flushes 2024-11-15T17:42:14,808 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:14,808 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:14,808 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. after waiting 0 ms 2024-11-15T17:42:14,808 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:14,813 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T17:42:14,814 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:14,814 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 248b48bc55c0fdcd5620a92777fad08f: Waiting for close lock at 1731692534808Running coprocessor pre-close hooks at 1731692534808Disabling compacts and flushes for region at 1731692534808Disabling writes for close at 1731692534808Writing region close event to WAL at 1731692534809 (+1 ms)Running coprocessor post-close hooks at 1731692534814 (+5 ms)Closed at 1731692534814 2024-11-15T17:42:14,814 WARN [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 248b48bc55c0fdcd5620a92777fad08f to self. 2024-11-15T17:42:14,817 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:14,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=CLOSED 2024-11-15T17:42:14,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:42:14,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-15T17:42:14,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 in 164 msec 2024-11-15T17:42:14,824 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, REOPEN/MOVE; state=CLOSED, location=4e3b17656000,45187,1731692383005; forceNewPlan=false, retain=true 2024-11-15T17:42:14,975 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=OPENING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:42:14,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, REOPEN/MOVE because future has completed 2024-11-15T17:42:14,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:42:15,139 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,140 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:42:15,141 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,141 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:42:15,141 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,141 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,143 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,144 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 248b48bc55c0fdcd5620a92777fad08f columnFamilyName A 2024-11-15T17:42:15,145 DEBUG [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:15,145 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(327): Store=248b48bc55c0fdcd5620a92777fad08f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:42:15,146 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,146 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 248b48bc55c0fdcd5620a92777fad08f columnFamilyName B 2024-11-15T17:42:15,146 DEBUG [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:15,147 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(327): Store=248b48bc55c0fdcd5620a92777fad08f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:42:15,147 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,147 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 248b48bc55c0fdcd5620a92777fad08f columnFamilyName C 2024-11-15T17:42:15,147 DEBUG [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:15,148 INFO [StoreOpener-248b48bc55c0fdcd5620a92777fad08f-1 {}] regionserver.HStore(327): Store=248b48bc55c0fdcd5620a92777fad08f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:42:15,148 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,149 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,150 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,151 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,151 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,151 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T17:42:15,152 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,153 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 248b48bc55c0fdcd5620a92777fad08f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69614177, jitterRate=0.03733207285404205}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T17:42:15,153 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,154 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 248b48bc55c0fdcd5620a92777fad08f: Running coprocessor pre-open hook at 1731692535141Writing region info on filesystem at 1731692535141Initializing all the Stores at 1731692535143 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692535143Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692535143Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692535143Cleaning up temporary data from old regions at 1731692535151 (+8 ms)Running coprocessor post-open hooks at 1731692535153 (+2 ms)Region opened successfully at 1731692535154 (+1 ms) 2024-11-15T17:42:15,155 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., pid=151, masterSystemTime=1731692535132 2024-11-15T17:42:15,156 DEBUG [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,157 INFO [RS_OPEN_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,157 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=OPEN, openSeqNum=5, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:42:15,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-15T17:42:15,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 in 180 msec 2024-11-15T17:42:15,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T17:42:15,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, REOPEN/MOVE in 509 msec 2024-11-15T17:42:15,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-15T17:42:15,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-15T17:42:15,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 935 msec 2024-11-15T17:42:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-15T17:42:15,170 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:15,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,172 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d535690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf5e76c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,177 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7114f4a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,179 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fad6749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,182 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686bd649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,183 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b7ea2f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,185 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be2bdec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51cbf0a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,188 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1796faca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea20cd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,192 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9f703d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,193 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24393e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,195 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e75e602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,197 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6d0bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,199 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b75edc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fad953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,202 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f6a419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,204 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ae47a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4e3b17656000,41205,-1 for getting cluster id 2024-11-15T17:42:15,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:42:15,206 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f848a115-bc41-4153-aa8f-4783b69d52ca' 2024-11-15T17:42:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:42:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f848a115-bc41-4153-aa8f-4783b69d52ca" 2024-11-15T17:42:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c13d97f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4e3b17656000,41205,-1] 2024-11-15T17:42:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:42:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:15,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:42:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21fbeacb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:42:15,209 DEBUG [Thread-2499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,209 DEBUG [Thread-2491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,209 DEBUG [Thread-2497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,209 DEBUG [Thread-2510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:42:15,209 DEBUG [Thread-2495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,210 DEBUG [Thread-2493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,211 DEBUG [Thread-2504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-15T17:42:15,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,212 DEBUG [Thread-2506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T17:42:15,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,214 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:42:15,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,214 DEBUG [Thread-2502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,215 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:42:15,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:42:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,216 DEBUG [Thread-2508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4e3b17656000,45187,1731692383005, seqNum=-1] 2024-11-15T17:42:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:42:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:42:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5] 2024-11-15T17:42:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,222 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:15,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T17:42:15,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692595240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692595242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692595243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692595245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692595246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115854311aa15474b3faa559821f07e1983_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692535221/Put/seqid=0 2024-11-15T17:42:15,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742386_1562 (size=12154) 2024-11-15T17:42:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T17:42:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T17:42:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T17:42:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T17:42:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692595565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692595565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692595565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692595565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692595565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,653 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:15,656 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115854311aa15474b3faa559821f07e1983_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115854311aa15474b3faa559821f07e1983_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:15,657 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/b35d22d416b54d0da69e66e01ac57597, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:15,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/b35d22d416b54d0da69e66e01ac57597 is 175, key is test_row_0/A:col10/1731692535221/Put/seqid=0 2024-11-15T17:42:15,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742387_1563 (size=30955) 2024-11-15T17:42:15,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T17:42:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T17:42:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T17:42:15,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:15,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T17:42:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:15,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:16,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/b35d22d416b54d0da69e66e01ac57597 2024-11-15T17:42:16,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/99e90c566c8241239076e1c3ce8329f8 is 50, key is test_row_0/B:col10/1731692535221/Put/seqid=0 2024-11-15T17:42:16,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742388_1564 (size=12001) 2024-11-15T17:42:16,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/99e90c566c8241239076e1c3ce8329f8 2024-11-15T17:42:16,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/a386a6055776438e96b993ab6fa33cf2 is 50, key is test_row_0/C:col10/1731692535221/Put/seqid=0 2024-11-15T17:42:16,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742389_1565 (size=12001) 2024-11-15T17:42:16,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/a386a6055776438e96b993ab6fa33cf2 2024-11-15T17:42:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/b35d22d416b54d0da69e66e01ac57597 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b35d22d416b54d0da69e66e01ac57597 2024-11-15T17:42:16,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b35d22d416b54d0da69e66e01ac57597, entries=150, sequenceid=16, filesize=30.2 K 2024-11-15T17:42:16,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/99e90c566c8241239076e1c3ce8329f8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/99e90c566c8241239076e1c3ce8329f8 2024-11-15T17:42:16,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/99e90c566c8241239076e1c3ce8329f8, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T17:42:16,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/a386a6055776438e96b993ab6fa33cf2 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a386a6055776438e96b993ab6fa33cf2 2024-11-15T17:42:16,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a386a6055776438e96b993ab6fa33cf2, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T17:42:16,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 248b48bc55c0fdcd5620a92777fad08f in 903ms, sequenceid=16, compaction requested=false 2024-11-15T17:42:16,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:16,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T17:42:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:16,132 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T17:42:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152e8062858ddc445bb5aa4769814cac6e_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692535223/Put/seqid=0 2024-11-15T17:42:16,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742390_1566 (size=12154) 2024-11-15T17:42:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T17:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:16,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:16,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692596388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692596389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692596391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692596392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692596392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692596496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692596496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692596505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692596505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692596505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:16,550 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152e8062858ddc445bb5aa4769814cac6e_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e8062858ddc445bb5aa4769814cac6e_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:16,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/f6e94b0b1a0e462385efd7d6b589ab4e, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/f6e94b0b1a0e462385efd7d6b589ab4e is 175, key is test_row_0/A:col10/1731692535223/Put/seqid=0 2024-11-15T17:42:16,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742391_1567 (size=30955) 2024-11-15T17:42:16,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692596706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692596706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692596715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692596715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692596715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:16,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:42:16,958 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/f6e94b0b1a0e462385efd7d6b589ab4e 2024-11-15T17:42:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/bd6962ddac0e482a8db6ce6d58145fd9 is 50, key is test_row_0/B:col10/1731692535223/Put/seqid=0 2024-11-15T17:42:16,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742392_1568 (size=12001) 2024-11-15T17:42:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692597016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692597016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692597025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692597025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692597025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T17:42:17,367 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/bd6962ddac0e482a8db6ce6d58145fd9 2024-11-15T17:42:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/6e26af28c8e64e9bb52ba39b841255f7 is 50, key is test_row_0/C:col10/1731692535223/Put/seqid=0 2024-11-15T17:42:17,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742393_1569 (size=12001) 2024-11-15T17:42:17,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692597525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692597525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692597535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692597536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:17,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692597536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:17,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:17,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:17,776 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/6e26af28c8e64e9bb52ba39b841255f7 2024-11-15T17:42:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/f6e94b0b1a0e462385efd7d6b589ab4e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/f6e94b0b1a0e462385efd7d6b589ab4e 2024-11-15T17:42:17,783 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/f6e94b0b1a0e462385efd7d6b589ab4e, entries=150, sequenceid=41, filesize=30.2 K 2024-11-15T17:42:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/bd6962ddac0e482a8db6ce6d58145fd9 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/bd6962ddac0e482a8db6ce6d58145fd9 2024-11-15T17:42:17,798 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/bd6962ddac0e482a8db6ce6d58145fd9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T17:42:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/6e26af28c8e64e9bb52ba39b841255f7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6e26af28c8e64e9bb52ba39b841255f7 2024-11-15T17:42:17,801 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6e26af28c8e64e9bb52ba39b841255f7, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T17:42:17,802 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 248b48bc55c0fdcd5620a92777fad08f in 1670ms, sequenceid=41, compaction requested=false 2024-11-15T17:42:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-15T17:42:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-15T17:42:17,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T17:42:17,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5880 sec 2024-11-15T17:42:17,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.5950 sec 2024-11-15T17:42:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:18,537 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:18,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T17:42:18,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cb683325080b42b3a90b048af039e886_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692538536/Put/seqid=0 2024-11-15T17:42:18,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742394_1570 (size=14594) 2024-11-15T17:42:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692598564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692598564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692598565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692598565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692598565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692598675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692598675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692598675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692598675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692598675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692598885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692598885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692598885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692598885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692598885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:18,947 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:18,949 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cb683325080b42b3a90b048af039e886_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cb683325080b42b3a90b048af039e886_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:18,950 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/fc6024ce7f844f7e8688d0cd6fd80ec6, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:18,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/fc6024ce7f844f7e8688d0cd6fd80ec6 is 175, key is test_row_0/A:col10/1731692538536/Put/seqid=0 2024-11-15T17:42:18,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742395_1571 (size=39549) 2024-11-15T17:42:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692599196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692599196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692599196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692599196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692599196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,359 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/fc6024ce7f844f7e8688d0cd6fd80ec6 2024-11-15T17:42:19,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4075947006a54b6c842d94937d6a1cd5 is 50, key is test_row_0/B:col10/1731692538536/Put/seqid=0 2024-11-15T17:42:19,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742396_1572 (size=12001) 2024-11-15T17:42:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T17:42:19,376 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:19,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:42:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-15T17:42:19,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:42:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T17:42:19,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:42:19,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:42:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T17:42:19,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T17:42:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T17:42:19,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T17:42:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692599705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692599705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692599705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692599706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692599706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:19,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4075947006a54b6c842d94937d6a1cd5 2024-11-15T17:42:19,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/242643c429bd42788dba4df1215fbc5e is 50, key is test_row_0/C:col10/1731692538536/Put/seqid=0 2024-11-15T17:42:19,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742397_1573 (size=12001) 2024-11-15T17:42:19,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T17:42:19,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:19,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T17:42:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:19,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T17:42:20,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T17:42:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:20,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:20,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/242643c429bd42788dba4df1215fbc5e 2024-11-15T17:42:20,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/fc6024ce7f844f7e8688d0cd6fd80ec6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/fc6024ce7f844f7e8688d0cd6fd80ec6 2024-11-15T17:42:20,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/fc6024ce7f844f7e8688d0cd6fd80ec6, entries=200, sequenceid=54, filesize=38.6 K 2024-11-15T17:42:20,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4075947006a54b6c842d94937d6a1cd5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4075947006a54b6c842d94937d6a1cd5 2024-11-15T17:42:20,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4075947006a54b6c842d94937d6a1cd5, entries=150, sequenceid=54, filesize=11.7 K 2024-11-15T17:42:20,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/242643c429bd42788dba4df1215fbc5e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/242643c429bd42788dba4df1215fbc5e 2024-11-15T17:42:20,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/242643c429bd42788dba4df1215fbc5e, entries=150, sequenceid=54, filesize=11.7 K 2024-11-15T17:42:20,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 248b48bc55c0fdcd5620a92777fad08f in 1655ms, sequenceid=54, compaction requested=true 2024-11-15T17:42:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:20,192 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:20,192 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/B is initiating minor compaction (all files) 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/A is initiating minor compaction (all files) 2024-11-15T17:42:20,193 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/B in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:20,193 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/A in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:20,193 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/99e90c566c8241239076e1c3ce8329f8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/bd6962ddac0e482a8db6ce6d58145fd9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4075947006a54b6c842d94937d6a1cd5] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.2 K 2024-11-15T17:42:20,193 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b35d22d416b54d0da69e66e01ac57597, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/f6e94b0b1a0e462385efd7d6b589ab4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/fc6024ce7f844f7e8688d0cd6fd80ec6] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=99.1 K 2024-11-15T17:42:20,193 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b35d22d416b54d0da69e66e01ac57597, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/f6e94b0b1a0e462385efd7d6b589ab4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/fc6024ce7f844f7e8688d0cd6fd80ec6] 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e90c566c8241239076e1c3ce8329f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692535217 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting bd6962ddac0e482a8db6ce6d58145fd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731692535223 2024-11-15T17:42:20,193 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting b35d22d416b54d0da69e66e01ac57597, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692535217 2024-11-15T17:42:20,194 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6e94b0b1a0e462385efd7d6b589ab4e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731692535223 2024-11-15T17:42:20,194 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4075947006a54b6c842d94937d6a1cd5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731692536388 2024-11-15T17:42:20,194 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6024ce7f844f7e8688d0cd6fd80ec6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731692536388 2024-11-15T17:42:20,199 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:20,200 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:20,200 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/6f784927c1624bdb8f6d841704bd4d41 is 50, key is test_row_0/B:col10/1731692538536/Put/seqid=0 2024-11-15T17:42:20,213 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a8c5a1ab6afe490289bc8d6b152d6199_248b48bc55c0fdcd5620a92777fad08f store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:20,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a8c5a1ab6afe490289bc8d6b152d6199_248b48bc55c0fdcd5620a92777fad08f, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:20,214 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a8c5a1ab6afe490289bc8d6b152d6199_248b48bc55c0fdcd5620a92777fad08f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:20,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742398_1574 (size=12104) 2024-11-15T17:42:20,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742399_1575 (size=4469) 2024-11-15T17:42:20,244 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#A#compaction#483 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:20,244 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/4302577721fd45029c30fb1611c0fce4 is 175, key is test_row_0/A:col10/1731692538536/Put/seqid=0 2024-11-15T17:42:20,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742400_1576 (size=31058) 2024-11-15T17:42:20,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T17:42:20,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:20,293 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T17:42:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111560f3472e15c24139855d7a25b165b768_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692538564/Put/seqid=0 2024-11-15T17:42:20,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742401_1577 (size=12154) 2024-11-15T17:42:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T17:42:20,627 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/6f784927c1624bdb8f6d841704bd4d41 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/6f784927c1624bdb8f6d841704bd4d41 2024-11-15T17:42:20,631 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/B of 248b48bc55c0fdcd5620a92777fad08f into 6f784927c1624bdb8f6d841704bd4d41(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:20,631 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:20,631 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/B, priority=13, startTime=1731692540192; duration=0sec 2024-11-15T17:42:20,631 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:20,631 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:B 2024-11-15T17:42:20,631 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:20,631 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:20,631 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/C is initiating minor compaction (all files) 2024-11-15T17:42:20,632 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/C in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:20,632 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a386a6055776438e96b993ab6fa33cf2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6e26af28c8e64e9bb52ba39b841255f7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/242643c429bd42788dba4df1215fbc5e] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.2 K 2024-11-15T17:42:20,632 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting a386a6055776438e96b993ab6fa33cf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731692535217 2024-11-15T17:42:20,632 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e26af28c8e64e9bb52ba39b841255f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731692535223 2024-11-15T17:42:20,632 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 242643c429bd42788dba4df1215fbc5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731692536388 2024-11-15T17:42:20,637 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:20,637 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/562465cc765440098983c7282d9d6138 is 50, key is test_row_0/C:col10/1731692538536/Put/seqid=0 2024-11-15T17:42:20,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742402_1578 (size=12104) 2024-11-15T17:42:20,647 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/562465cc765440098983c7282d9d6138 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/562465cc765440098983c7282d9d6138 2024-11-15T17:42:20,652 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/C of 248b48bc55c0fdcd5620a92777fad08f into 562465cc765440098983c7282d9d6138(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:20,652 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:20,652 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/C, priority=13, startTime=1731692540192; duration=0sec 2024-11-15T17:42:20,652 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:20,652 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:C 2024-11-15T17:42:20,652 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/4302577721fd45029c30fb1611c0fce4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/4302577721fd45029c30fb1611c0fce4 2024-11-15T17:42:20,656 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/A of 248b48bc55c0fdcd5620a92777fad08f into 4302577721fd45029c30fb1611c0fce4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:20,656 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:20,656 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/A, priority=13, startTime=1731692540192; duration=0sec 2024-11-15T17:42:20,656 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:20,656 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:A 2024-11-15T17:42:20,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:20,704 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111560f3472e15c24139855d7a25b165b768_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560f3472e15c24139855d7a25b165b768_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/9ff4e7d84b504e93b9166816119258b4, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/9ff4e7d84b504e93b9166816119258b4 is 175, key is test_row_0/A:col10/1731692538564/Put/seqid=0 2024-11-15T17:42:20,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742403_1579 (size=30955) 2024-11-15T17:42:20,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:20,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692600722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692600723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692600724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692600726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692600726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692600836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692600836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692600836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692600836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692600836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692601046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692601046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692601046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692601046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692601046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,109 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/9ff4e7d84b504e93b9166816119258b4 2024-11-15T17:42:21,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/74ce4c35b49e4eda988cb7e2067a74ee is 50, key is test_row_0/B:col10/1731692538564/Put/seqid=0 2024-11-15T17:42:21,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742404_1580 (size=12001) 2024-11-15T17:42:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692601355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692601355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692601355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692601355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692601355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T17:42:21,526 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/74ce4c35b49e4eda988cb7e2067a74ee 2024-11-15T17:42:21,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/0475517162fd487b88523cb0a1077a13 is 50, key is test_row_0/C:col10/1731692538564/Put/seqid=0 2024-11-15T17:42:21,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742405_1581 (size=12001) 2024-11-15T17:42:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692601865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692601865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692601866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692601866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692601866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:21,935 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/0475517162fd487b88523cb0a1077a13 2024-11-15T17:42:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/9ff4e7d84b504e93b9166816119258b4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/9ff4e7d84b504e93b9166816119258b4 2024-11-15T17:42:21,941 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/9ff4e7d84b504e93b9166816119258b4, entries=150, sequenceid=77, filesize=30.2 K 2024-11-15T17:42:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/74ce4c35b49e4eda988cb7e2067a74ee as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/74ce4c35b49e4eda988cb7e2067a74ee 2024-11-15T17:42:21,944 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/74ce4c35b49e4eda988cb7e2067a74ee, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T17:42:21,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/0475517162fd487b88523cb0a1077a13 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/0475517162fd487b88523cb0a1077a13 2024-11-15T17:42:21,949 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/0475517162fd487b88523cb0a1077a13, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T17:42:21,950 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 248b48bc55c0fdcd5620a92777fad08f in 1658ms, sequenceid=77, compaction requested=false 2024-11-15T17:42:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-15T17:42:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-15T17:42:21,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-15T17:42:21,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5740 sec 2024-11-15T17:42:21,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.5780 sec 2024-11-15T17:42:22,540 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T17:42:22,540 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T17:42:22,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:22,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T17:42:22,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152fcd1086fe304577af96b07e378b0828_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692540721/Put/seqid=0 2024-11-15T17:42:22,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742406_1582 (size=14594) 2024-11-15T17:42:22,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692602897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692602899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692602899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692602900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:22,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692602901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692603005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692603005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692603005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692603016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692603016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692603215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692603215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692603216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692603226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692603226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,288 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:23,291 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152fcd1086fe304577af96b07e378b0828_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152fcd1086fe304577af96b07e378b0828_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:23,292 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/ef1f3f9716c4436d942cf0a044c06240, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:23,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/ef1f3f9716c4436d942cf0a044c06240 is 175, key is test_row_0/A:col10/1731692540721/Put/seqid=0 2024-11-15T17:42:23,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742407_1583 (size=39549) 2024-11-15T17:42:23,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692603525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692603525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692603525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692603535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692603536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T17:42:23,546 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:23,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:42:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-15T17:42:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T17:42:23,548 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:42:23,549 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:42:23,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:42:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T17:42:23,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T17:42:23,704 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/ef1f3f9716c4436d942cf0a044c06240 2024-11-15T17:42:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:23,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:23,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/44e24925cb0e4e7caf29de8e31b6ab12 is 50, key is test_row_0/B:col10/1731692540721/Put/seqid=0 2024-11-15T17:42:23,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742408_1584 (size=12001) 2024-11-15T17:42:23,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T17:42:23,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:23,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:23,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:23,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:23,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T17:42:24,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T17:42:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692604036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:24,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692604036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692604036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:24,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692604045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:24,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692604045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:24,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/44e24925cb0e4e7caf29de8e31b6ab12 2024-11-15T17:42:24,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/ef411f7e861e4bb0bdbc21787e70acc0 is 50, key is test_row_0/C:col10/1731692540721/Put/seqid=0 2024-11-15T17:42:24,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742409_1585 (size=12001) 2024-11-15T17:42:24,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T17:42:24,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:24,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T17:42:24,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T17:42:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T17:42:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:24,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/ef411f7e861e4bb0bdbc21787e70acc0 2024-11-15T17:42:24,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/ef1f3f9716c4436d942cf0a044c06240 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/ef1f3f9716c4436d942cf0a044c06240 2024-11-15T17:42:24,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/ef1f3f9716c4436d942cf0a044c06240, entries=200, sequenceid=94, filesize=38.6 K 2024-11-15T17:42:24,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/44e24925cb0e4e7caf29de8e31b6ab12 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/44e24925cb0e4e7caf29de8e31b6ab12 2024-11-15T17:42:24,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/44e24925cb0e4e7caf29de8e31b6ab12, entries=150, sequenceid=94, filesize=11.7 K 2024-11-15T17:42:24,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/ef411f7e861e4bb0bdbc21787e70acc0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/ef411f7e861e4bb0bdbc21787e70acc0 2024-11-15T17:42:24,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/ef411f7e861e4bb0bdbc21787e70acc0, entries=150, sequenceid=94, filesize=11.7 K 2024-11-15T17:42:24,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 248b48bc55c0fdcd5620a92777fad08f in 1657ms, sequenceid=94, compaction requested=true 2024-11-15T17:42:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:24,536 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:24,536 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:24,537 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:24,537 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:24,537 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/A is initiating minor compaction (all files) 2024-11-15T17:42:24,537 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/B is initiating minor compaction (all files) 2024-11-15T17:42:24,537 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/A in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,537 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/B in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,537 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/6f784927c1624bdb8f6d841704bd4d41, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/74ce4c35b49e4eda988cb7e2067a74ee, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/44e24925cb0e4e7caf29de8e31b6ab12] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.3 K 2024-11-15T17:42:24,537 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/4302577721fd45029c30fb1611c0fce4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/9ff4e7d84b504e93b9166816119258b4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/ef1f3f9716c4436d942cf0a044c06240] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=99.2 K 2024-11-15T17:42:24,537 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,537 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/4302577721fd45029c30fb1611c0fce4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/9ff4e7d84b504e93b9166816119258b4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/ef1f3f9716c4436d942cf0a044c06240] 2024-11-15T17:42:24,538 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f784927c1624bdb8f6d841704bd4d41, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731692536388 2024-11-15T17:42:24,538 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4302577721fd45029c30fb1611c0fce4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731692536388 2024-11-15T17:42:24,538 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff4e7d84b504e93b9166816119258b4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731692538563 2024-11-15T17:42:24,538 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ce4c35b49e4eda988cb7e2067a74ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731692538563 2024-11-15T17:42:24,538 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1f3f9716c4436d942cf0a044c06240, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692540721 2024-11-15T17:42:24,538 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 44e24925cb0e4e7caf29de8e31b6ab12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692540721 2024-11-15T17:42:24,543 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:24,545 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#B#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:24,545 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f8db762175e44573a277cdff40b9f1f3 is 50, key is test_row_0/B:col10/1731692540721/Put/seqid=0 2024-11-15T17:42:24,547 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ffb8f47a07bc43079500a669ed793772_248b48bc55c0fdcd5620a92777fad08f store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:24,549 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ffb8f47a07bc43079500a669ed793772_248b48bc55c0fdcd5620a92777fad08f, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:24,549 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ffb8f47a07bc43079500a669ed793772_248b48bc55c0fdcd5620a92777fad08f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:24,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742410_1586 (size=12207) 2024-11-15T17:42:24,555 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f8db762175e44573a277cdff40b9f1f3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f8db762175e44573a277cdff40b9f1f3 2024-11-15T17:42:24,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742411_1587 (size=4469) 2024-11-15T17:42:24,560 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/B of 248b48bc55c0fdcd5620a92777fad08f into f8db762175e44573a277cdff40b9f1f3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:24,560 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:24,560 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/B, priority=13, startTime=1731692544536; duration=0sec 2024-11-15T17:42:24,560 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:24,560 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:B 2024-11-15T17:42:24,560 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:24,560 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:24,561 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/C is initiating minor compaction (all files) 2024-11-15T17:42:24,561 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/C in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,561 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/562465cc765440098983c7282d9d6138, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/0475517162fd487b88523cb0a1077a13, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/ef411f7e861e4bb0bdbc21787e70acc0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.3 K 2024-11-15T17:42:24,561 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 562465cc765440098983c7282d9d6138, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731692536388 2024-11-15T17:42:24,561 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 0475517162fd487b88523cb0a1077a13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731692538563 2024-11-15T17:42:24,561 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting ef411f7e861e4bb0bdbc21787e70acc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692540721 2024-11-15T17:42:24,567 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:24,567 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/8e20150f808b4d27a8bbe4a550d2ad31 is 50, key is test_row_0/C:col10/1731692540721/Put/seqid=0 2024-11-15T17:42:24,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742412_1588 (size=12207) 2024-11-15T17:42:24,573 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/8e20150f808b4d27a8bbe4a550d2ad31 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/8e20150f808b4d27a8bbe4a550d2ad31 2024-11-15T17:42:24,581 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/C of 248b48bc55c0fdcd5620a92777fad08f into 8e20150f808b4d27a8bbe4a550d2ad31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:24,581 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:24,581 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/C, priority=13, startTime=1731692544536; duration=0sec 2024-11-15T17:42:24,581 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:24,581 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:C 2024-11-15T17:42:24,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T17:42:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:24,618 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T17:42:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115da3101d69ef2430781b9e1ca9e756c0b_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692542898/Put/seqid=0 2024-11-15T17:42:24,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742413_1589 (size=12154) 2024-11-15T17:42:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T17:42:24,958 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:24,958 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/330d5fe1b46f49599ab34389e619791d is 175, key is test_row_0/A:col10/1731692540721/Put/seqid=0 2024-11-15T17:42:24,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742414_1590 (size=31161) 2024-11-15T17:42:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:25,029 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115da3101d69ef2430781b9e1ca9e756c0b_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115da3101d69ef2430781b9e1ca9e756c0b_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/7304350f41154073a421b1799f8b9b6a, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:25,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/7304350f41154073a421b1799f8b9b6a is 175, key is test_row_0/A:col10/1731692542898/Put/seqid=0 2024-11-15T17:42:25,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742415_1591 (size=30955) 2024-11-15T17:42:25,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:25,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692605059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692605060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692605060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692605061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692605062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692605165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692605175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692605175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692605175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692605176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,365 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/330d5fe1b46f49599ab34389e619791d as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/330d5fe1b46f49599ab34389e619791d 2024-11-15T17:42:25,368 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/A of 248b48bc55c0fdcd5620a92777fad08f into 330d5fe1b46f49599ab34389e619791d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:25,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:25,368 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/A, priority=13, startTime=1731692544536; duration=0sec 2024-11-15T17:42:25,368 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:25,369 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:A 2024-11-15T17:42:25,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692605375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692605385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692605386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692605386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692605386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,438 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/7304350f41154073a421b1799f8b9b6a 2024-11-15T17:42:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/80a751ea9ec243b892f03e93bf35f075 is 50, key is test_row_0/B:col10/1731692542898/Put/seqid=0 2024-11-15T17:42:25,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742416_1592 (size=12001) 2024-11-15T17:42:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692605685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692605695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692605695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692605695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692605696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T17:42:25,848 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/80a751ea9ec243b892f03e93bf35f075 2024-11-15T17:42:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/743c37b3c2034af6858265b1f2e1379c is 50, key is test_row_0/C:col10/1731692542898/Put/seqid=0 2024-11-15T17:42:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742417_1593 (size=12001) 2024-11-15T17:42:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692606195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692606210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692606210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692606210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692606210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:26,256 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/743c37b3c2034af6858265b1f2e1379c 2024-11-15T17:42:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/7304350f41154073a421b1799f8b9b6a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/7304350f41154073a421b1799f8b9b6a 2024-11-15T17:42:26,262 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/7304350f41154073a421b1799f8b9b6a, entries=150, sequenceid=118, filesize=30.2 K 2024-11-15T17:42:26,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/80a751ea9ec243b892f03e93bf35f075 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/80a751ea9ec243b892f03e93bf35f075 2024-11-15T17:42:26,265 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/80a751ea9ec243b892f03e93bf35f075, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T17:42:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/743c37b3c2034af6858265b1f2e1379c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/743c37b3c2034af6858265b1f2e1379c 2024-11-15T17:42:26,270 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/743c37b3c2034af6858265b1f2e1379c, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T17:42:26,271 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 248b48bc55c0fdcd5620a92777fad08f in 1653ms, sequenceid=118, compaction requested=false 2024-11-15T17:42:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-15T17:42:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-15T17:42:26,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-15T17:42:26,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7230 sec 2024-11-15T17:42:26,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.7280 sec 2024-11-15T17:42:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:27,218 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:27,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T17:42:27,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d228fefb90a3405789bca01ff85a0a40_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692545060/Put/seqid=0 2024-11-15T17:42:27,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742418_1594 (size=12254) 2024-11-15T17:42:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692607238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692607239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692607239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692607240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692607241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692607346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692607346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692607355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692607355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692607355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692607556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692607556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692607565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692607565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692607565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:27,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d228fefb90a3405789bca01ff85a0a40_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d228fefb90a3405789bca01ff85a0a40_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:27,633 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/14822773f9044c498f6b1f57528396d8, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:27,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/14822773f9044c498f6b1f57528396d8 is 175, key is test_row_0/A:col10/1731692545060/Put/seqid=0 2024-11-15T17:42:27,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742419_1595 (size=31055) 2024-11-15T17:42:27,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/14822773f9044c498f6b1f57528396d8 2024-11-15T17:42:27,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f7a1f8bdaadf42e4a526bf994009eb70 is 50, key is test_row_0/B:col10/1731692545060/Put/seqid=0 2024-11-15T17:42:27,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742420_1596 (size=12101) 2024-11-15T17:42:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T17:42:27,726 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:27,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:42:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-15T17:42:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:42:27,728 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:42:27,728 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:42:27,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:42:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:42:27,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692607866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692607866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692607875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692607875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692607875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:27,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:27,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:27,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:42:28,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:42:28,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f7a1f8bdaadf42e4a526bf994009eb70 2024-11-15T17:42:28,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/b5defca83d574be99254c5a7ad6660e8 is 50, key is test_row_0/C:col10/1731692545060/Put/seqid=0 2024-11-15T17:42:28,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742421_1597 (size=12101) 2024-11-15T17:42:28,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:42:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:42:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:42:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692608375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692608375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:28,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692608385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:28,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692608385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692608386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:28,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/b5defca83d574be99254c5a7ad6660e8 2024-11-15T17:42:28,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/14822773f9044c498f6b1f57528396d8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14822773f9044c498f6b1f57528396d8 2024-11-15T17:42:28,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14822773f9044c498f6b1f57528396d8, entries=150, sequenceid=134, filesize=30.3 K 2024-11-15T17:42:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f7a1f8bdaadf42e4a526bf994009eb70 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f7a1f8bdaadf42e4a526bf994009eb70 2024-11-15T17:42:28,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f7a1f8bdaadf42e4a526bf994009eb70, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T17:42:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/b5defca83d574be99254c5a7ad6660e8 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/b5defca83d574be99254c5a7ad6660e8 2024-11-15T17:42:28,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/b5defca83d574be99254c5a7ad6660e8, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T17:42:28,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 248b48bc55c0fdcd5620a92777fad08f in 1250ms, sequenceid=134, compaction requested=true 2024-11-15T17:42:28,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:28,469 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:28,469 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:28,469 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:28,469 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:28,469 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/B is initiating minor compaction (all files) 2024-11-15T17:42:28,469 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/A is initiating minor compaction (all files) 2024-11-15T17:42:28,470 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/A in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,470 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/B in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,470 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/330d5fe1b46f49599ab34389e619791d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/7304350f41154073a421b1799f8b9b6a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14822773f9044c498f6b1f57528396d8] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=91.0 K 2024-11-15T17:42:28,470 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f8db762175e44573a277cdff40b9f1f3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/80a751ea9ec243b892f03e93bf35f075, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f7a1f8bdaadf42e4a526bf994009eb70] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.5 K 2024-11-15T17:42:28,470 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,470 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/330d5fe1b46f49599ab34389e619791d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/7304350f41154073a421b1799f8b9b6a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14822773f9044c498f6b1f57528396d8] 2024-11-15T17:42:28,470 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 330d5fe1b46f49599ab34389e619791d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692540721 2024-11-15T17:42:28,470 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f8db762175e44573a277cdff40b9f1f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692540721 2024-11-15T17:42:28,470 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7304350f41154073a421b1799f8b9b6a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731692542891 2024-11-15T17:42:28,470 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a751ea9ec243b892f03e93bf35f075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731692542891 2024-11-15T17:42:28,471 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14822773f9044c498f6b1f57528396d8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692545059 2024-11-15T17:42:28,471 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a1f8bdaadf42e4a526bf994009eb70, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692545059 2024-11-15T17:42:28,474 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:28,475 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:28,476 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111560621ae965fa4b5e954730fa66b688c2_248b48bc55c0fdcd5620a92777fad08f store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:28,476 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/217fb6fccd924b8d86c06d237e9edb83 is 50, key is test_row_0/B:col10/1731692545060/Put/seqid=0 2024-11-15T17:42:28,477 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111560621ae965fa4b5e954730fa66b688c2_248b48bc55c0fdcd5620a92777fad08f, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:28,477 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111560621ae965fa4b5e954730fa66b688c2_248b48bc55c0fdcd5620a92777fad08f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:28,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742422_1598 (size=12409) 2024-11-15T17:42:28,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742423_1599 (size=4469) 2024-11-15T17:42:28,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T17:42:28,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,490 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T17:42:28,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fe9aacc69ac441c0b3d09809d4b04ec0_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692547239/Put/seqid=0 2024-11-15T17:42:28,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742424_1600 (size=12304) 2024-11-15T17:42:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:42:28,885 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#A#compaction#502 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:28,886 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/1fe3ec53b58b47cf89f1d378a85aebb4 is 175, key is test_row_0/A:col10/1731692545060/Put/seqid=0 2024-11-15T17:42:28,887 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/217fb6fccd924b8d86c06d237e9edb83 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/217fb6fccd924b8d86c06d237e9edb83 2024-11-15T17:42:28,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742425_1601 (size=31363) 2024-11-15T17:42:28,891 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/B of 248b48bc55c0fdcd5620a92777fad08f into 217fb6fccd924b8d86c06d237e9edb83(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:28,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:28,891 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/B, priority=13, startTime=1731692548469; duration=0sec 2024-11-15T17:42:28,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:28,891 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:B 2024-11-15T17:42:28,892 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:28,892 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:28,892 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/C is initiating minor compaction (all files) 2024-11-15T17:42:28,892 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/C in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:28,893 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/8e20150f808b4d27a8bbe4a550d2ad31, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/743c37b3c2034af6858265b1f2e1379c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/b5defca83d574be99254c5a7ad6660e8] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.5 K 2024-11-15T17:42:28,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e20150f808b4d27a8bbe4a550d2ad31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731692540721 2024-11-15T17:42:28,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 743c37b3c2034af6858265b1f2e1379c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731692542891 2024-11-15T17:42:28,893 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting b5defca83d574be99254c5a7ad6660e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692545059 2024-11-15T17:42:28,898 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:28,899 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/6f454e700b404fd790ad8189d5affb05 is 50, key is test_row_0/C:col10/1731692545060/Put/seqid=0 2024-11-15T17:42:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:28,902 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fe9aacc69ac441c0b3d09809d4b04ec0_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fe9aacc69ac441c0b3d09809d4b04ec0_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:28,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/e9062172324d4594a0516adb8de2fbc6, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:28,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/e9062172324d4594a0516adb8de2fbc6 is 175, key is test_row_0/A:col10/1731692547239/Put/seqid=0 2024-11-15T17:42:28,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742426_1602 (size=12409) 2024-11-15T17:42:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742427_1603 (size=31105) 2024-11-15T17:42:29,297 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/1fe3ec53b58b47cf89f1d378a85aebb4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/1fe3ec53b58b47cf89f1d378a85aebb4 2024-11-15T17:42:29,303 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/A of 248b48bc55c0fdcd5620a92777fad08f into 1fe3ec53b58b47cf89f1d378a85aebb4(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:29,303 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:29,303 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/A, priority=13, startTime=1731692548469; duration=0sec 2024-11-15T17:42:29,303 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:29,303 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:A 2024-11-15T17:42:29,314 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/e9062172324d4594a0516adb8de2fbc6 2024-11-15T17:42:29,314 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/6f454e700b404fd790ad8189d5affb05 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6f454e700b404fd790ad8189d5affb05 2024-11-15T17:42:29,320 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/C of 248b48bc55c0fdcd5620a92777fad08f into 6f454e700b404fd790ad8189d5affb05(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:29,320 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:29,320 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/C, priority=13, startTime=1731692548469; duration=0sec 2024-11-15T17:42:29,320 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:29,320 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:C 2024-11-15T17:42:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4d4574e2079a49539ddf7f225f6e3c86 is 50, key is test_row_0/B:col10/1731692547239/Put/seqid=0 2024-11-15T17:42:29,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742428_1604 (size=12151) 2024-11-15T17:42:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:29,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:29,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692609398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692609399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692609400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692609400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692609401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692609506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692609506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692609506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692609506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692609506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692609715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692609715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692609715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692609715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692609715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:29,726 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4d4574e2079a49539ddf7f225f6e3c86 2024-11-15T17:42:29,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/cca83875018c4c5c96d5bf587be1e382 is 50, key is test_row_0/C:col10/1731692547239/Put/seqid=0 2024-11-15T17:42:29,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742429_1605 (size=12151) 2024-11-15T17:42:29,746 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/cca83875018c4c5c96d5bf587be1e382 2024-11-15T17:42:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/e9062172324d4594a0516adb8de2fbc6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/e9062172324d4594a0516adb8de2fbc6 2024-11-15T17:42:29,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/e9062172324d4594a0516adb8de2fbc6, entries=150, sequenceid=155, filesize=30.4 K 2024-11-15T17:42:29,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4d4574e2079a49539ddf7f225f6e3c86 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4d4574e2079a49539ddf7f225f6e3c86 2024-11-15T17:42:29,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4d4574e2079a49539ddf7f225f6e3c86, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T17:42:29,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/cca83875018c4c5c96d5bf587be1e382 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/cca83875018c4c5c96d5bf587be1e382 2024-11-15T17:42:29,770 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/cca83875018c4c5c96d5bf587be1e382, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T17:42:29,771 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 248b48bc55c0fdcd5620a92777fad08f in 1281ms, sequenceid=155, compaction requested=false 2024-11-15T17:42:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-15T17:42:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-15T17:42:29,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T17:42:29,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0440 sec 2024-11-15T17:42:29,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.0490 sec 2024-11-15T17:42:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T17:42:29,876 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:29,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:42:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-15T17:42:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:42:29,878 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:42:29,878 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:42:29,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:42:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:42:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:30,028 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:30,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T17:42:30,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:42:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115687da4b06afb4f62ace56b3bc8753aba_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692549398/Put/seqid=0 2024-11-15T17:42:30,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742430_1606 (size=14794) 2024-11-15T17:42:30,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:30,040 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115687da4b06afb4f62ace56b3bc8753aba_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115687da4b06afb4f62ace56b3bc8753aba_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:30,041 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/95958e1b3d55415797b0207c1bbfb134, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:30,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/95958e1b3d55415797b0207c1bbfb134 is 175, key is test_row_0/A:col10/1731692549398/Put/seqid=0 2024-11-15T17:42:30,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742431_1607 (size=39749) 2024-11-15T17:42:30,045 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/95958e1b3d55415797b0207c1bbfb134 2024-11-15T17:42:30,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692610043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692610043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692610046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692610047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/19dd2f50a27b41018807b78b10666165 is 50, key is test_row_0/B:col10/1731692549398/Put/seqid=0 2024-11-15T17:42:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692610048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742432_1608 (size=12151) 2024-11-15T17:42:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692610155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692610156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692610156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692610156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692610156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:42:30,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:30,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:42:30,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:42:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692610366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692610366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692610366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692610366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692610366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/19dd2f50a27b41018807b78b10666165 2024-11-15T17:42:30,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/81470cc0d394477ea376bc1e301297ab is 50, key is test_row_0/C:col10/1731692549398/Put/seqid=0 2024-11-15T17:42:30,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742433_1609 (size=12151) 2024-11-15T17:42:30,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:42:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:42:30,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692610676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692610676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692610676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692610676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692610676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:30,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:42:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:30,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/81470cc0d394477ea376bc1e301297ab 2024-11-15T17:42:30,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/95958e1b3d55415797b0207c1bbfb134 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/95958e1b3d55415797b0207c1bbfb134 2024-11-15T17:42:30,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/95958e1b3d55415797b0207c1bbfb134, entries=200, sequenceid=174, filesize=38.8 K 2024-11-15T17:42:30,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/19dd2f50a27b41018807b78b10666165 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/19dd2f50a27b41018807b78b10666165 2024-11-15T17:42:30,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/19dd2f50a27b41018807b78b10666165, entries=150, sequenceid=174, filesize=11.9 K 2024-11-15T17:42:30,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/81470cc0d394477ea376bc1e301297ab as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/81470cc0d394477ea376bc1e301297ab 2024-11-15T17:42:30,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/81470cc0d394477ea376bc1e301297ab, entries=150, sequenceid=174, filesize=11.9 K 2024-11-15T17:42:30,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 248b48bc55c0fdcd5620a92777fad08f in 851ms, sequenceid=174, compaction requested=true 2024-11-15T17:42:30,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:30,879 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:30,880 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:30,880 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:30,880 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/A is initiating minor compaction (all files) 2024-11-15T17:42:30,880 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/A in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,880 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/1fe3ec53b58b47cf89f1d378a85aebb4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/e9062172324d4594a0516adb8de2fbc6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/95958e1b3d55415797b0207c1bbfb134] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=99.8 K 2024-11-15T17:42:30,880 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:30,880 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,880 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/B is initiating minor compaction (all files) 2024-11-15T17:42:30,880 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/1fe3ec53b58b47cf89f1d378a85aebb4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/e9062172324d4594a0516adb8de2fbc6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/95958e1b3d55415797b0207c1bbfb134] 2024-11-15T17:42:30,880 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/B in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,880 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/217fb6fccd924b8d86c06d237e9edb83, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4d4574e2079a49539ddf7f225f6e3c86, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/19dd2f50a27b41018807b78b10666165] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.9 K 2024-11-15T17:42:30,881 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 217fb6fccd924b8d86c06d237e9edb83, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692545059 2024-11-15T17:42:30,881 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fe3ec53b58b47cf89f1d378a85aebb4, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692545059 2024-11-15T17:42:30,881 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9062172324d4594a0516adb8de2fbc6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692547233 2024-11-15T17:42:30,881 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4574e2079a49539ddf7f225f6e3c86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692547233 2024-11-15T17:42:30,881 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 19dd2f50a27b41018807b78b10666165, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731692549398 2024-11-15T17:42:30,881 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95958e1b3d55415797b0207c1bbfb134, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731692549398 2024-11-15T17:42:30,886 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:30,887 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:30,888 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/42b425ccaa2b46a4870990147da0b985 is 50, key is test_row_0/B:col10/1731692549398/Put/seqid=0 2024-11-15T17:42:30,906 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411155945991dd5eb4ea3a67bf0f5992b5a0f_248b48bc55c0fdcd5620a92777fad08f store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:30,908 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411155945991dd5eb4ea3a67bf0f5992b5a0f_248b48bc55c0fdcd5620a92777fad08f, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:30,909 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155945991dd5eb4ea3a67bf0f5992b5a0f_248b48bc55c0fdcd5620a92777fad08f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742434_1610 (size=12561) 2024-11-15T17:42:30,915 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/42b425ccaa2b46a4870990147da0b985 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/42b425ccaa2b46a4870990147da0b985 2024-11-15T17:42:30,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742435_1611 (size=4469) 2024-11-15T17:42:30,920 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/B of 248b48bc55c0fdcd5620a92777fad08f into 42b425ccaa2b46a4870990147da0b985(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:30,920 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:30,920 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/B, priority=13, startTime=1731692550880; duration=0sec 2024-11-15T17:42:30,920 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:30,920 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:B 2024-11-15T17:42:30,920 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:30,921 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:30,921 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/C is initiating minor compaction (all files) 2024-11-15T17:42:30,921 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/C in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,921 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6f454e700b404fd790ad8189d5affb05, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/cca83875018c4c5c96d5bf587be1e382, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/81470cc0d394477ea376bc1e301297ab] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=35.9 K 2024-11-15T17:42:30,921 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f454e700b404fd790ad8189d5affb05, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731692545059 2024-11-15T17:42:30,922 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting cca83875018c4c5c96d5bf587be1e382, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731692547233 2024-11-15T17:42:30,922 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 81470cc0d394477ea376bc1e301297ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731692549398 2024-11-15T17:42:30,929 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#C#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:30,929 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/5194b807762246878d90c82e5928e1e5 is 50, key is test_row_0/C:col10/1731692549398/Put/seqid=0 2024-11-15T17:42:30,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742436_1612 (size=12561) 2024-11-15T17:42:30,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T17:42:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:30,944 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:42:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115373bb10202a64ada8d23ae688e0d96cc_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692550042/Put/seqid=0 2024-11-15T17:42:30,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742437_1613 (size=12304) 2024-11-15T17:42:30,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:30,969 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115373bb10202a64ada8d23ae688e0d96cc_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115373bb10202a64ada8d23ae688e0d96cc_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/cef2c29ee02440139488b8dec017f734, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:30,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/cef2c29ee02440139488b8dec017f734 is 175, key is test_row_0/A:col10/1731692550042/Put/seqid=0 2024-11-15T17:42:30,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742438_1614 (size=31105) 2024-11-15T17:42:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:42:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:31,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692611196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692611199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692611226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692611226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692611227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,320 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#A#compaction#511 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:31,321 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/b124182aa6754d438b797057c8673878 is 175, key is test_row_0/A:col10/1731692549398/Put/seqid=0 2024-11-15T17:42:31,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742439_1615 (size=31515) 2024-11-15T17:42:31,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692611336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692611336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692611336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692611336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692611336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,341 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/5194b807762246878d90c82e5928e1e5 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5194b807762246878d90c82e5928e1e5 2024-11-15T17:42:31,345 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/C of 248b48bc55c0fdcd5620a92777fad08f into 5194b807762246878d90c82e5928e1e5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:31,345 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:31,345 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/C, priority=13, startTime=1731692550880; duration=0sec 2024-11-15T17:42:31,345 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:31,345 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:C 2024-11-15T17:42:31,373 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/cef2c29ee02440139488b8dec017f734 2024-11-15T17:42:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f6bfd202fd934e3098cba180c2e7836a is 50, key is test_row_0/B:col10/1731692550042/Put/seqid=0 2024-11-15T17:42:31,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742440_1616 (size=12151) 2024-11-15T17:42:31,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692611546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692611546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692611546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692611546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692611546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,736 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/b124182aa6754d438b797057c8673878 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b124182aa6754d438b797057c8673878 2024-11-15T17:42:31,741 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/A of 248b48bc55c0fdcd5620a92777fad08f into b124182aa6754d438b797057c8673878(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:31,741 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:31,741 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/A, priority=13, startTime=1731692550879; duration=0sec 2024-11-15T17:42:31,741 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:31,741 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:A 2024-11-15T17:42:31,782 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f6bfd202fd934e3098cba180c2e7836a 2024-11-15T17:42:31,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/5c9d2da2b5674758983cf22350eb647a is 50, key is test_row_0/C:col10/1731692550042/Put/seqid=0 2024-11-15T17:42:31,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742441_1617 (size=12151) 2024-11-15T17:42:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692611855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692611855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692611855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692611856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:31,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692611856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:42:32,204 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/5c9d2da2b5674758983cf22350eb647a 2024-11-15T17:42:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/cef2c29ee02440139488b8dec017f734 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/cef2c29ee02440139488b8dec017f734 2024-11-15T17:42:32,210 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/cef2c29ee02440139488b8dec017f734, entries=150, sequenceid=195, filesize=30.4 K 2024-11-15T17:42:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/f6bfd202fd934e3098cba180c2e7836a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f6bfd202fd934e3098cba180c2e7836a 2024-11-15T17:42:32,213 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f6bfd202fd934e3098cba180c2e7836a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T17:42:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/5c9d2da2b5674758983cf22350eb647a as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5c9d2da2b5674758983cf22350eb647a 2024-11-15T17:42:32,216 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5c9d2da2b5674758983cf22350eb647a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T17:42:32,217 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for 248b48bc55c0fdcd5620a92777fad08f in 1273ms, sequenceid=195, compaction requested=false 2024-11-15T17:42:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-15T17:42:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-15T17:42:32,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T17:42:32,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3390 sec 2024-11-15T17:42:32,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.3440 sec 2024-11-15T17:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:32,368 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:32,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T17:42:32,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b94de083afa347f4b24c752cf00bbbfa_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692551199/Put/seqid=0 2024-11-15T17:42:32,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742442_1618 (size=14794) 2024-11-15T17:42:32,377 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:32,381 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b94de083afa347f4b24c752cf00bbbfa_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b94de083afa347f4b24c752cf00bbbfa_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:32,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/027bf5e55ef74d19a55530cab2014091, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:32,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/027bf5e55ef74d19a55530cab2014091 is 175, key is test_row_0/A:col10/1731692551199/Put/seqid=0 2024-11-15T17:42:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692612379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692612380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692612380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692612382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692612383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742443_1619 (size=39749) 2024-11-15T17:42:32,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692612496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692612496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692612496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692612496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692612496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692612706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692612706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692612706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692612706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692612706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:32,794 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/027bf5e55ef74d19a55530cab2014091 2024-11-15T17:42:32,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/9e4e0fc9efbf4a6baaaff8d526916cf1 is 50, key is test_row_0/B:col10/1731692551199/Put/seqid=0 2024-11-15T17:42:32,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742444_1620 (size=12151) 2024-11-15T17:42:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692613015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692613015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692613015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692613015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692613015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/9e4e0fc9efbf4a6baaaff8d526916cf1 2024-11-15T17:42:33,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/e3a56bcf2cc2412e805052d4bbf44cd6 is 50, key is test_row_0/C:col10/1731692551199/Put/seqid=0 2024-11-15T17:42:33,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742445_1621 (size=12151) 2024-11-15T17:42:33,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/e3a56bcf2cc2412e805052d4bbf44cd6 2024-11-15T17:42:33,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/027bf5e55ef74d19a55530cab2014091 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/027bf5e55ef74d19a55530cab2014091 2024-11-15T17:42:33,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/027bf5e55ef74d19a55530cab2014091, entries=200, sequenceid=218, filesize=38.8 K 2024-11-15T17:42:33,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/9e4e0fc9efbf4a6baaaff8d526916cf1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/9e4e0fc9efbf4a6baaaff8d526916cf1 2024-11-15T17:42:33,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/9e4e0fc9efbf4a6baaaff8d526916cf1, entries=150, sequenceid=218, filesize=11.9 K 2024-11-15T17:42:33,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/e3a56bcf2cc2412e805052d4bbf44cd6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/e3a56bcf2cc2412e805052d4bbf44cd6 2024-11-15T17:42:33,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/e3a56bcf2cc2412e805052d4bbf44cd6, entries=150, sequenceid=218, filesize=11.9 K 2024-11-15T17:42:33,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 248b48bc55c0fdcd5620a92777fad08f in 875ms, sequenceid=218, compaction requested=true 2024-11-15T17:42:33,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:33,244 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:33,244 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:33,245 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:33,245 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/B is initiating minor compaction (all files) 2024-11-15T17:42:33,245 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/B in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:33,245 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/42b425ccaa2b46a4870990147da0b985, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f6bfd202fd934e3098cba180c2e7836a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/9e4e0fc9efbf4a6baaaff8d526916cf1] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=36.0 K 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/A is initiating minor compaction (all files) 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b425ccaa2b46a4870990147da0b985, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731692549398 2024-11-15T17:42:33,246 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/A in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:33,246 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b124182aa6754d438b797057c8673878, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/cef2c29ee02440139488b8dec017f734, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/027bf5e55ef74d19a55530cab2014091] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=100.0 K 2024-11-15T17:42:33,246 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b124182aa6754d438b797057c8673878, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/cef2c29ee02440139488b8dec017f734, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/027bf5e55ef74d19a55530cab2014091] 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting f6bfd202fd934e3098cba180c2e7836a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731692550042 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e4e0fc9efbf4a6baaaff8d526916cf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692551199 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting b124182aa6754d438b797057c8673878, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731692549398 2024-11-15T17:42:33,246 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef2c29ee02440139488b8dec017f734, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731692550042 2024-11-15T17:42:33,247 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027bf5e55ef74d19a55530cab2014091, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692551199 2024-11-15T17:42:33,253 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:33,255 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:33,256 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/65b2e62a207d47639414714d6c61f094 is 50, key is test_row_0/B:col10/1731692551199/Put/seqid=0 2024-11-15T17:42:33,256 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e1f24bdd083c4f388eb862c10dd01da1_248b48bc55c0fdcd5620a92777fad08f store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:33,258 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e1f24bdd083c4f388eb862c10dd01da1_248b48bc55c0fdcd5620a92777fad08f, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:33,258 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e1f24bdd083c4f388eb862c10dd01da1_248b48bc55c0fdcd5620a92777fad08f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:33,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742446_1622 (size=12663) 2024-11-15T17:42:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742447_1623 (size=4469) 2024-11-15T17:42:33,285 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#A#compaction#519 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:33,287 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/23636c1ef17c401e8bcfb980b346293c is 175, key is test_row_0/A:col10/1731692551199/Put/seqid=0 2024-11-15T17:42:33,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742448_1624 (size=31617) 2024-11-15T17:42:33,303 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/23636c1ef17c401e8bcfb980b346293c as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/23636c1ef17c401e8bcfb980b346293c 2024-11-15T17:42:33,317 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/A of 248b48bc55c0fdcd5620a92777fad08f into 23636c1ef17c401e8bcfb980b346293c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:33,317 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:33,317 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/A, priority=13, startTime=1731692553243; duration=0sec 2024-11-15T17:42:33,317 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:33,317 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:A 2024-11-15T17:42:33,317 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:33,318 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:33,318 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/C is initiating minor compaction (all files) 2024-11-15T17:42:33,318 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/C in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:33,318 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5194b807762246878d90c82e5928e1e5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5c9d2da2b5674758983cf22350eb647a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/e3a56bcf2cc2412e805052d4bbf44cd6] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=36.0 K 2024-11-15T17:42:33,319 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5194b807762246878d90c82e5928e1e5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731692549398 2024-11-15T17:42:33,319 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c9d2da2b5674758983cf22350eb647a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731692550042 2024-11-15T17:42:33,320 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3a56bcf2cc2412e805052d4bbf44cd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692551199 2024-11-15T17:42:33,329 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#C#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:33,329 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/1904c36a39884a77ba8686d46fc52106 is 50, key is test_row_0/C:col10/1731692551199/Put/seqid=0 2024-11-15T17:42:33,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742449_1625 (size=12663) 2024-11-15T17:42:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:33,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:33,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T17:42:33,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a357ed9c10e643e4bbe0b2306834ec61_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692552369/Put/seqid=0 2024-11-15T17:42:33,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692613543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692613543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692613544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692613547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692613546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742450_1626 (size=17284) 2024-11-15T17:42:33,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:33,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a357ed9c10e643e4bbe0b2306834ec61_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a357ed9c10e643e4bbe0b2306834ec61_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:33,564 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/45c5b90c9840414c97311788c2c7d2c1, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:33,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/45c5b90c9840414c97311788c2c7d2c1 is 175, key is test_row_0/A:col10/1731692552369/Put/seqid=0 2024-11-15T17:42:33,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742451_1627 (size=48389) 2024-11-15T17:42:33,574 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/45c5b90c9840414c97311788c2c7d2c1 2024-11-15T17:42:33,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4b8157b21ba24547bd22975667ea97b1 is 50, key is test_row_0/B:col10/1731692552369/Put/seqid=0 2024-11-15T17:42:33,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742452_1628 (size=12151) 2024-11-15T17:42:33,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4b8157b21ba24547bd22975667ea97b1 2024-11-15T17:42:33,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/a38077616639487c9656d4456f0028e4 is 50, key is test_row_0/C:col10/1731692552369/Put/seqid=0 2024-11-15T17:42:33,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742453_1629 (size=12151) 2024-11-15T17:42:33,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692613655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692613655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692613655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692613655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692613655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,668 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/65b2e62a207d47639414714d6c61f094 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/65b2e62a207d47639414714d6c61f094 2024-11-15T17:42:33,672 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/B of 248b48bc55c0fdcd5620a92777fad08f into 65b2e62a207d47639414714d6c61f094(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:33,672 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:33,672 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/B, priority=13, startTime=1731692553244; duration=0sec 2024-11-15T17:42:33,672 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:33,673 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:B 2024-11-15T17:42:33,742 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/1904c36a39884a77ba8686d46fc52106 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1904c36a39884a77ba8686d46fc52106 2024-11-15T17:42:33,747 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/C of 248b48bc55c0fdcd5620a92777fad08f into 1904c36a39884a77ba8686d46fc52106(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:33,747 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:33,747 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/C, priority=13, startTime=1731692553244; duration=0sec 2024-11-15T17:42:33,747 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:33,747 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:C 2024-11-15T17:42:33,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692613865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692613865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692613865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692613866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:33,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692613876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:33,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:33,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T17:42:34,045 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:34,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T17:42:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-15T17:42:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:42:34,047 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T17:42:34,048 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T17:42:34,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T17:42:34,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/a38077616639487c9656d4456f0028e4 2024-11-15T17:42:34,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/45c5b90c9840414c97311788c2c7d2c1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/45c5b90c9840414c97311788c2c7d2c1 2024-11-15T17:42:34,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/45c5b90c9840414c97311788c2c7d2c1, entries=250, sequenceid=237, filesize=47.3 K 2024-11-15T17:42:34,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/4b8157b21ba24547bd22975667ea97b1 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4b8157b21ba24547bd22975667ea97b1 2024-11-15T17:42:34,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4b8157b21ba24547bd22975667ea97b1, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T17:42:34,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/a38077616639487c9656d4456f0028e4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a38077616639487c9656d4456f0028e4 2024-11-15T17:42:34,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a38077616639487c9656d4456f0028e4, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T17:42:34,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 248b48bc55c0fdcd5620a92777fad08f in 535ms, sequenceid=237, compaction requested=false 2024-11-15T17:42:34,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:42:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:34,177 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T17:42:34,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T17:42:34,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e9e603b9f247429e82599988af09db9d_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692554176/Put/seqid=0 2024-11-15T17:42:34,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742454_1630 (size=12354) 2024-11-15T17:42:34,187 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:34,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e9e603b9f247429e82599988af09db9d_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e9e603b9f247429e82599988af09db9d_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:34,190 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/54c026f3ed7e4869bd5de07dba856012, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:34,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/54c026f3ed7e4869bd5de07dba856012 is 175, key is test_row_0/A:col10/1731692554176/Put/seqid=0 2024-11-15T17:42:34,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742455_1631 (size=31155) 2024-11-15T17:42:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692614192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692614192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692614193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692614197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:42:34,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:34,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692614197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692614305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692614305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692614305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692614305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692614306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:42:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:42:34,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:42:34,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:34,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692614515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692614515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692614515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692614515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692614515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,597 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/54c026f3ed7e4869bd5de07dba856012 2024-11-15T17:42:34,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/1a0b784c89904557ab5e882c70ce8f83 is 50, key is test_row_0/B:col10/1731692554176/Put/seqid=0 2024-11-15T17:42:34,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742456_1632 (size=12201) 2024-11-15T17:42:34,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/1a0b784c89904557ab5e882c70ce8f83 2024-11-15T17:42:34,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/1d56e16128784a2092aa097a78dde7e0 is 50, key is test_row_0/C:col10/1731692554176/Put/seqid=0 2024-11-15T17:42:34,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742457_1633 (size=12201) 2024-11-15T17:42:34,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:42:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:42:34,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:42:34,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:34,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55294 deadline: 1731692614826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55308 deadline: 1731692614826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55296 deadline: 1731692614826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55272 deadline: 1731692614826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T17:42:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45187 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55274 deadline: 1731692614826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 2024-11-15T17:42:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 , the old value is region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T17:42:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., hostname=4e3b17656000,45187,1731692383005, seqNum=5 because the exception is null or not the one we care about 2024-11-15T17:42:34,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:42:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:34,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:42:35,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/1d56e16128784a2092aa097a78dde7e0 2024-11-15T17:42:35,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/54c026f3ed7e4869bd5de07dba856012 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/54c026f3ed7e4869bd5de07dba856012 2024-11-15T17:42:35,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/54c026f3ed7e4869bd5de07dba856012, entries=150, sequenceid=258, filesize=30.4 K 2024-11-15T17:42:35,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/1a0b784c89904557ab5e882c70ce8f83 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/1a0b784c89904557ab5e882c70ce8f83 2024-11-15T17:42:35,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/1a0b784c89904557ab5e882c70ce8f83, entries=150, sequenceid=258, filesize=11.9 K 2024-11-15T17:42:35,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/1d56e16128784a2092aa097a78dde7e0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1d56e16128784a2092aa097a78dde7e0 2024-11-15T17:42:35,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1d56e16128784a2092aa097a78dde7e0, entries=150, sequenceid=258, filesize=11.9 K 2024-11-15T17:42:35,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 248b48bc55c0fdcd5620a92777fad08f in 865ms, sequenceid=258, compaction requested=true 2024-11-15T17:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T17:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:35,042 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T17:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:35,042 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 248b48bc55c0fdcd5620a92777fad08f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T17:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:35,043 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:35,043 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/A is initiating minor compaction (all files) 2024-11-15T17:42:35,043 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/A in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:35,043 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/23636c1ef17c401e8bcfb980b346293c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/45c5b90c9840414c97311788c2c7d2c1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/54c026f3ed7e4869bd5de07dba856012] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=108.6 K 2024-11-15T17:42:35,043 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:35,043 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. files: [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/23636c1ef17c401e8bcfb980b346293c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/45c5b90c9840414c97311788c2c7d2c1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/54c026f3ed7e4869bd5de07dba856012] 2024-11-15T17:42:35,044 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23636c1ef17c401e8bcfb980b346293c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692551199 2024-11-15T17:42:35,044 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c5b90c9840414c97311788c2c7d2c1, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731692552369 2024-11-15T17:42:35,044 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:35,044 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/B is initiating minor compaction (all files) 2024-11-15T17:42:35,044 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/B in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:35,044 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/65b2e62a207d47639414714d6c61f094, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4b8157b21ba24547bd22975667ea97b1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/1a0b784c89904557ab5e882c70ce8f83] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=36.1 K 2024-11-15T17:42:35,044 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c026f3ed7e4869bd5de07dba856012, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692553544 2024-11-15T17:42:35,044 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 65b2e62a207d47639414714d6c61f094, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692551199 2024-11-15T17:42:35,044 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8157b21ba24547bd22975667ea97b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731692552369 2024-11-15T17:42:35,045 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a0b784c89904557ab5e882c70ce8f83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692553544 2024-11-15T17:42:35,049 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:35,052 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#B#compaction#529 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:35,052 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/ab1485d8fb16430d93465fb11c97b653 is 50, key is test_row_0/B:col10/1731692554176/Put/seqid=0 2024-11-15T17:42:35,059 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111522e77ca7109d4a4f952ccc2a37f62f2f_248b48bc55c0fdcd5620a92777fad08f store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:35,060 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111522e77ca7109d4a4f952ccc2a37f62f2f_248b48bc55c0fdcd5620a92777fad08f, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:35,060 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111522e77ca7109d4a4f952ccc2a37f62f2f_248b48bc55c0fdcd5620a92777fad08f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:35,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742458_1634 (size=12815) 2024-11-15T17:42:35,075 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/ab1485d8fb16430d93465fb11c97b653 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/ab1485d8fb16430d93465fb11c97b653 2024-11-15T17:42:35,080 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/B of 248b48bc55c0fdcd5620a92777fad08f into ab1485d8fb16430d93465fb11c97b653(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:35,080 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:35,080 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/B, priority=13, startTime=1731692555042; duration=0sec 2024-11-15T17:42:35,080 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T17:42:35,080 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:B 2024-11-15T17:42:35,080 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T17:42:35,081 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T17:42:35,081 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1541): 248b48bc55c0fdcd5620a92777fad08f/C is initiating minor compaction (all files) 2024-11-15T17:42:35,081 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 248b48bc55c0fdcd5620a92777fad08f/C in TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:35,081 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1904c36a39884a77ba8686d46fc52106, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a38077616639487c9656d4456f0028e4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1d56e16128784a2092aa097a78dde7e0] into tmpdir=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp, totalSize=36.1 K 2024-11-15T17:42:35,082 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1904c36a39884a77ba8686d46fc52106, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731692551199 2024-11-15T17:42:35,082 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting a38077616639487c9656d4456f0028e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731692552369 2024-11-15T17:42:35,082 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d56e16128784a2092aa097a78dde7e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731692553544 2024-11-15T17:42:35,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742459_1635 (size=4469) 2024-11-15T17:42:35,085 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#A#compaction#528 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:35,086 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/14fb676415e84239939e5310c8be69a0 is 175, key is test_row_0/A:col10/1731692554176/Put/seqid=0 2024-11-15T17:42:35,091 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 248b48bc55c0fdcd5620a92777fad08f#C#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T17:42:35,091 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/70f1e5f939a54a4daa48db3375c08df3 is 50, key is test_row_0/C:col10/1731692554176/Put/seqid=0 2024-11-15T17:42:35,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742460_1636 (size=31769) 2024-11-15T17:42:35,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45187 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T17:42:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:35,115 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T17:42:35,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742461_1637 (size=12815) 2024-11-15T17:42:35,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156f85ddb30f704eb08b583d29a39af88c_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692554192/Put/seqid=0 2024-11-15T17:42:35,127 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/70f1e5f939a54a4daa48db3375c08df3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/70f1e5f939a54a4daa48db3375c08df3 2024-11-15T17:42:35,131 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/C of 248b48bc55c0fdcd5620a92777fad08f into 70f1e5f939a54a4daa48db3375c08df3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:35,131 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:35,131 INFO [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/C, priority=13, startTime=1731692555042; duration=0sec 2024-11-15T17:42:35,131 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:35,131 DEBUG [RS:0;4e3b17656000:45187-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:C 2024-11-15T17:42:35,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742462_1638 (size=12454) 2024-11-15T17:42:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:35,151 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156f85ddb30f704eb08b583d29a39af88c_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156f85ddb30f704eb08b583d29a39af88c_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/6bd5684443c745d79a61efee07a67444, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/6bd5684443c745d79a61efee07a67444 is 175, key is test_row_0/A:col10/1731692554192/Put/seqid=0 2024-11-15T17:42:35,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742463_1639 (size=31255) 2024-11-15T17:42:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:42:35,214 INFO [Thread-2510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2510. 2024-11-15T17:42:35,214 DEBUG [Thread-2510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,214 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,214 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,215 INFO [Thread-2508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2508. 2024-11-15T17:42:35,215 DEBUG [Thread-2508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,215 DEBUG [Thread-2508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,215 DEBUG [Thread-2508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,216 INFO [Thread-2504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2504. 2024-11-15T17:42:35,216 DEBUG [Thread-2504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,216 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,216 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,216 INFO [Thread-2506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2506. 2024-11-15T17:42:35,217 DEBUG [Thread-2506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,217 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,217 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,218 INFO [Thread-2502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2502. 2024-11-15T17:42:35,218 DEBUG [Thread-2502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,218 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,218 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(8855): Flush requested on 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:35,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. as already flushing 2024-11-15T17:42:35,337 INFO [Thread-2491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2491. 2024-11-15T17:42:35,337 INFO [Thread-2495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2495. 2024-11-15T17:42:35,337 INFO [Thread-2493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2493. 2024-11-15T17:42:35,337 DEBUG [Thread-2493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,337 DEBUG [Thread-2495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,337 DEBUG [Thread-2491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,337 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,337 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,337 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,338 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,338 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,338 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,339 INFO [Thread-2497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2497. 2024-11-15T17:42:35,339 DEBUG [Thread-2497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,339 DEBUG [Thread-2497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,339 DEBUG [Thread-2497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,341 INFO [Thread-2499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2499. 2024-11-15T17:42:35,341 DEBUG [Thread-2499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T17:42:35,341 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,341 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:35,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:35,516 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/14fb676415e84239939e5310c8be69a0 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14fb676415e84239939e5310c8be69a0 2024-11-15T17:42:35,523 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 248b48bc55c0fdcd5620a92777fad08f/A of 248b48bc55c0fdcd5620a92777fad08f into 14fb676415e84239939e5310c8be69a0(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T17:42:35,523 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:35,523 INFO [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., storeName=248b48bc55c0fdcd5620a92777fad08f/A, priority=13, startTime=1731692555042; duration=0sec 2024-11-15T17:42:35,523 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T17:42:35,524 DEBUG [RS:0;4e3b17656000:45187-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 248b48bc55c0fdcd5620a92777fad08f:A 2024-11-15T17:42:35,572 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/6bd5684443c745d79a61efee07a67444 2024-11-15T17:42:35,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/fc815a86fdcc4051a241de0d0f0c96e3 is 50, key is test_row_0/B:col10/1731692554192/Put/seqid=0 2024-11-15T17:42:35,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742464_1640 (size=12301) 2024-11-15T17:42:35,985 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/fc815a86fdcc4051a241de0d0f0c96e3 2024-11-15T17:42:35,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/aa76a8e5217a40b3b7395f715fd8c6a7 is 50, key is test_row_0/C:col10/1731692554192/Put/seqid=0 2024-11-15T17:42:35,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742465_1641 (size=12301) 2024-11-15T17:42:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:42:36,398 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/aa76a8e5217a40b3b7395f715fd8c6a7 2024-11-15T17:42:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/6bd5684443c745d79a61efee07a67444 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/6bd5684443c745d79a61efee07a67444 2024-11-15T17:42:36,412 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/6bd5684443c745d79a61efee07a67444, entries=150, sequenceid=277, filesize=30.5 K 2024-11-15T17:42:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/fc815a86fdcc4051a241de0d0f0c96e3 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/fc815a86fdcc4051a241de0d0f0c96e3 2024-11-15T17:42:36,415 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/fc815a86fdcc4051a241de0d0f0c96e3, entries=150, sequenceid=277, filesize=12.0 K 2024-11-15T17:42:36,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/aa76a8e5217a40b3b7395f715fd8c6a7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/aa76a8e5217a40b3b7395f715fd8c6a7 2024-11-15T17:42:36,417 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/aa76a8e5217a40b3b7395f715fd8c6a7, entries=150, sequenceid=277, filesize=12.0 K 2024-11-15T17:42:36,418 INFO [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=33.54 KB/34350 for 248b48bc55c0fdcd5620a92777fad08f in 1303ms, sequenceid=277, compaction requested=false 2024-11-15T17:42:36,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 248b48bc55c0fdcd5620a92777fad08f: 2024-11-15T17:42:36,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:36,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4e3b17656000:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-15T17:42:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-15T17:42:36,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-15T17:42:36,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3710 sec 2024-11-15T17:42:36,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.3750 sec 2024-11-15T17:42:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T17:42:38,207 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4443 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4197 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4179 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4433 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4191 2024-11-15T17:42:38,207 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T17:42:38,208 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:42:38,208 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:42:38,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:38,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:38,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:38,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T17:42:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:42:38,218 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692558218"}]},"ts":"1731692558218"} 2024-11-15T17:42:38,219 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T17:42:38,220 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T17:42:38,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T17:42:38,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, UNASSIGN}] 2024-11-15T17:42:38,222 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, UNASSIGN 2024-11-15T17:42:38,223 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=CLOSING, regionLocation=4e3b17656000,45187,1731692383005 2024-11-15T17:42:38,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, UNASSIGN because future has completed 2024-11-15T17:42:38,225 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T17:42:38,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005}] 2024-11-15T17:42:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:42:38,378 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:38,378 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T17:42:38,379 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 248b48bc55c0fdcd5620a92777fad08f, disabling compactions & flushes 2024-11-15T17:42:38,379 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:38,379 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:38,379 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. after waiting 0 ms 2024-11-15T17:42:38,379 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:38,379 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 248b48bc55c0fdcd5620a92777fad08f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T17:42:38,389 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115accdd202159e4e8fb0d88153b8718a7c_248b48bc55c0fdcd5620a92777fad08f is 50, key is test_row_0/A:col10/1731692555337/Put/seqid=0 2024-11-15T17:42:38,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742466_1642 (size=9914) 2024-11-15T17:42:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:42:38,797 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:42:38,803 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115accdd202159e4e8fb0d88153b8718a7c_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115accdd202159e4e8fb0d88153b8718a7c_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:38,804 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/0a529e13645d456fb734eec09e66e76b, store: [table=TestAcidGuarantees family=A region=248b48bc55c0fdcd5620a92777fad08f] 2024-11-15T17:42:38,805 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/0a529e13645d456fb734eec09e66e76b is 175, key is test_row_0/A:col10/1731692555337/Put/seqid=0 2024-11-15T17:42:38,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742467_1643 (size=22561) 2024-11-15T17:42:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:42:39,212 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/0a529e13645d456fb734eec09e66e76b 2024-11-15T17:42:39,222 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/e5d696addc9e43eeadd4a09d33fad4c4 is 50, key is test_row_0/B:col10/1731692555337/Put/seqid=0 2024-11-15T17:42:39,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742468_1644 (size=9857) 2024-11-15T17:42:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:42:39,628 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/e5d696addc9e43eeadd4a09d33fad4c4 2024-11-15T17:42:39,640 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/4df463b121624310a8814e025df86b40 is 50, key is test_row_0/C:col10/1731692555337/Put/seqid=0 2024-11-15T17:42:39,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742469_1645 (size=9857) 2024-11-15T17:42:40,044 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/4df463b121624310a8814e025df86b40 2024-11-15T17:42:40,049 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/A/0a529e13645d456fb734eec09e66e76b as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/0a529e13645d456fb734eec09e66e76b 2024-11-15T17:42:40,054 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/0a529e13645d456fb734eec09e66e76b, entries=100, sequenceid=287, filesize=22.0 K 2024-11-15T17:42:40,055 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/B/e5d696addc9e43eeadd4a09d33fad4c4 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/e5d696addc9e43eeadd4a09d33fad4c4 2024-11-15T17:42:40,058 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/e5d696addc9e43eeadd4a09d33fad4c4, entries=100, sequenceid=287, filesize=9.6 K 2024-11-15T17:42:40,059 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/.tmp/C/4df463b121624310a8814e025df86b40 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/4df463b121624310a8814e025df86b40 2024-11-15T17:42:40,062 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/4df463b121624310a8814e025df86b40, entries=100, sequenceid=287, filesize=9.6 K 2024-11-15T17:42:40,063 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 248b48bc55c0fdcd5620a92777fad08f in 1683ms, sequenceid=287, compaction requested=true 2024-11-15T17:42:40,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b35d22d416b54d0da69e66e01ac57597, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/f6e94b0b1a0e462385efd7d6b589ab4e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/fc6024ce7f844f7e8688d0cd6fd80ec6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/4302577721fd45029c30fb1611c0fce4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/9ff4e7d84b504e93b9166816119258b4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/ef1f3f9716c4436d942cf0a044c06240, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/330d5fe1b46f49599ab34389e619791d, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/7304350f41154073a421b1799f8b9b6a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/1fe3ec53b58b47cf89f1d378a85aebb4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14822773f9044c498f6b1f57528396d8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/e9062172324d4594a0516adb8de2fbc6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/95958e1b3d55415797b0207c1bbfb134, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b124182aa6754d438b797057c8673878, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/cef2c29ee02440139488b8dec017f734, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/027bf5e55ef74d19a55530cab2014091, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/23636c1ef17c401e8bcfb980b346293c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/45c5b90c9840414c97311788c2c7d2c1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/54c026f3ed7e4869bd5de07dba856012] to archive 2024-11-15T17:42:40,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:42:40,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b35d22d416b54d0da69e66e01ac57597 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b35d22d416b54d0da69e66e01ac57597 2024-11-15T17:42:40,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/f6e94b0b1a0e462385efd7d6b589ab4e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/f6e94b0b1a0e462385efd7d6b589ab4e 2024-11-15T17:42:40,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/fc6024ce7f844f7e8688d0cd6fd80ec6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/fc6024ce7f844f7e8688d0cd6fd80ec6 2024-11-15T17:42:40,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/4302577721fd45029c30fb1611c0fce4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/4302577721fd45029c30fb1611c0fce4 2024-11-15T17:42:40,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/9ff4e7d84b504e93b9166816119258b4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/9ff4e7d84b504e93b9166816119258b4 2024-11-15T17:42:40,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/ef1f3f9716c4436d942cf0a044c06240 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/ef1f3f9716c4436d942cf0a044c06240 2024-11-15T17:42:40,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/330d5fe1b46f49599ab34389e619791d to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/330d5fe1b46f49599ab34389e619791d 2024-11-15T17:42:40,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/7304350f41154073a421b1799f8b9b6a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/7304350f41154073a421b1799f8b9b6a 2024-11-15T17:42:40,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/1fe3ec53b58b47cf89f1d378a85aebb4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/1fe3ec53b58b47cf89f1d378a85aebb4 2024-11-15T17:42:40,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14822773f9044c498f6b1f57528396d8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14822773f9044c498f6b1f57528396d8 2024-11-15T17:42:40,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/e9062172324d4594a0516adb8de2fbc6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/e9062172324d4594a0516adb8de2fbc6 2024-11-15T17:42:40,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/95958e1b3d55415797b0207c1bbfb134 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/95958e1b3d55415797b0207c1bbfb134 2024-11-15T17:42:40,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b124182aa6754d438b797057c8673878 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/b124182aa6754d438b797057c8673878 2024-11-15T17:42:40,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/cef2c29ee02440139488b8dec017f734 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/cef2c29ee02440139488b8dec017f734 2024-11-15T17:42:40,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/027bf5e55ef74d19a55530cab2014091 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/027bf5e55ef74d19a55530cab2014091 2024-11-15T17:42:40,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/23636c1ef17c401e8bcfb980b346293c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/23636c1ef17c401e8bcfb980b346293c 2024-11-15T17:42:40,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/45c5b90c9840414c97311788c2c7d2c1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/45c5b90c9840414c97311788c2c7d2c1 2024-11-15T17:42:40,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/54c026f3ed7e4869bd5de07dba856012 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/54c026f3ed7e4869bd5de07dba856012 2024-11-15T17:42:40,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/99e90c566c8241239076e1c3ce8329f8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/bd6962ddac0e482a8db6ce6d58145fd9, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/6f784927c1624bdb8f6d841704bd4d41, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4075947006a54b6c842d94937d6a1cd5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/74ce4c35b49e4eda988cb7e2067a74ee, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f8db762175e44573a277cdff40b9f1f3, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/44e24925cb0e4e7caf29de8e31b6ab12, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/80a751ea9ec243b892f03e93bf35f075, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/217fb6fccd924b8d86c06d237e9edb83, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f7a1f8bdaadf42e4a526bf994009eb70, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4d4574e2079a49539ddf7f225f6e3c86, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/42b425ccaa2b46a4870990147da0b985, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/19dd2f50a27b41018807b78b10666165, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f6bfd202fd934e3098cba180c2e7836a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/65b2e62a207d47639414714d6c61f094, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/9e4e0fc9efbf4a6baaaff8d526916cf1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4b8157b21ba24547bd22975667ea97b1, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/1a0b784c89904557ab5e882c70ce8f83] to archive 2024-11-15T17:42:40,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:42:40,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/99e90c566c8241239076e1c3ce8329f8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/99e90c566c8241239076e1c3ce8329f8 2024-11-15T17:42:40,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/bd6962ddac0e482a8db6ce6d58145fd9 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/bd6962ddac0e482a8db6ce6d58145fd9 2024-11-15T17:42:40,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/6f784927c1624bdb8f6d841704bd4d41 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/6f784927c1624bdb8f6d841704bd4d41 2024-11-15T17:42:40,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4075947006a54b6c842d94937d6a1cd5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4075947006a54b6c842d94937d6a1cd5 2024-11-15T17:42:40,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/74ce4c35b49e4eda988cb7e2067a74ee to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/74ce4c35b49e4eda988cb7e2067a74ee 2024-11-15T17:42:40,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f8db762175e44573a277cdff40b9f1f3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f8db762175e44573a277cdff40b9f1f3 2024-11-15T17:42:40,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/44e24925cb0e4e7caf29de8e31b6ab12 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/44e24925cb0e4e7caf29de8e31b6ab12 2024-11-15T17:42:40,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/80a751ea9ec243b892f03e93bf35f075 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/80a751ea9ec243b892f03e93bf35f075 2024-11-15T17:42:40,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/217fb6fccd924b8d86c06d237e9edb83 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/217fb6fccd924b8d86c06d237e9edb83 2024-11-15T17:42:40,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f7a1f8bdaadf42e4a526bf994009eb70 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f7a1f8bdaadf42e4a526bf994009eb70 2024-11-15T17:42:40,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4d4574e2079a49539ddf7f225f6e3c86 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4d4574e2079a49539ddf7f225f6e3c86 2024-11-15T17:42:40,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/42b425ccaa2b46a4870990147da0b985 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/42b425ccaa2b46a4870990147da0b985 2024-11-15T17:42:40,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/19dd2f50a27b41018807b78b10666165 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/19dd2f50a27b41018807b78b10666165 2024-11-15T17:42:40,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f6bfd202fd934e3098cba180c2e7836a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/f6bfd202fd934e3098cba180c2e7836a 2024-11-15T17:42:40,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/65b2e62a207d47639414714d6c61f094 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/65b2e62a207d47639414714d6c61f094 2024-11-15T17:42:40,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/9e4e0fc9efbf4a6baaaff8d526916cf1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/9e4e0fc9efbf4a6baaaff8d526916cf1 2024-11-15T17:42:40,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4b8157b21ba24547bd22975667ea97b1 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/4b8157b21ba24547bd22975667ea97b1 2024-11-15T17:42:40,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/1a0b784c89904557ab5e882c70ce8f83 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/1a0b784c89904557ab5e882c70ce8f83 2024-11-15T17:42:40,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a386a6055776438e96b993ab6fa33cf2, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6e26af28c8e64e9bb52ba39b841255f7, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/562465cc765440098983c7282d9d6138, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/242643c429bd42788dba4df1215fbc5e, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/0475517162fd487b88523cb0a1077a13, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/8e20150f808b4d27a8bbe4a550d2ad31, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/ef411f7e861e4bb0bdbc21787e70acc0, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/743c37b3c2034af6858265b1f2e1379c, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6f454e700b404fd790ad8189d5affb05, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/b5defca83d574be99254c5a7ad6660e8, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/cca83875018c4c5c96d5bf587be1e382, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5194b807762246878d90c82e5928e1e5, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/81470cc0d394477ea376bc1e301297ab, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5c9d2da2b5674758983cf22350eb647a, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1904c36a39884a77ba8686d46fc52106, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/e3a56bcf2cc2412e805052d4bbf44cd6, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a38077616639487c9656d4456f0028e4, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1d56e16128784a2092aa097a78dde7e0] to archive 2024-11-15T17:42:40,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T17:42:40,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a386a6055776438e96b993ab6fa33cf2 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a386a6055776438e96b993ab6fa33cf2 2024-11-15T17:42:40,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6e26af28c8e64e9bb52ba39b841255f7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6e26af28c8e64e9bb52ba39b841255f7 2024-11-15T17:42:40,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/562465cc765440098983c7282d9d6138 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/562465cc765440098983c7282d9d6138 2024-11-15T17:42:40,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/242643c429bd42788dba4df1215fbc5e to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/242643c429bd42788dba4df1215fbc5e 2024-11-15T17:42:40,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/0475517162fd487b88523cb0a1077a13 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/0475517162fd487b88523cb0a1077a13 2024-11-15T17:42:40,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/8e20150f808b4d27a8bbe4a550d2ad31 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/8e20150f808b4d27a8bbe4a550d2ad31 2024-11-15T17:42:40,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/ef411f7e861e4bb0bdbc21787e70acc0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/ef411f7e861e4bb0bdbc21787e70acc0 2024-11-15T17:42:40,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/743c37b3c2034af6858265b1f2e1379c to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/743c37b3c2034af6858265b1f2e1379c 2024-11-15T17:42:40,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6f454e700b404fd790ad8189d5affb05 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/6f454e700b404fd790ad8189d5affb05 2024-11-15T17:42:40,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/b5defca83d574be99254c5a7ad6660e8 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/b5defca83d574be99254c5a7ad6660e8 2024-11-15T17:42:40,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/cca83875018c4c5c96d5bf587be1e382 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/cca83875018c4c5c96d5bf587be1e382 2024-11-15T17:42:40,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5194b807762246878d90c82e5928e1e5 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5194b807762246878d90c82e5928e1e5 2024-11-15T17:42:40,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/81470cc0d394477ea376bc1e301297ab to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/81470cc0d394477ea376bc1e301297ab 2024-11-15T17:42:40,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5c9d2da2b5674758983cf22350eb647a to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/5c9d2da2b5674758983cf22350eb647a 2024-11-15T17:42:40,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1904c36a39884a77ba8686d46fc52106 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1904c36a39884a77ba8686d46fc52106 2024-11-15T17:42:40,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/e3a56bcf2cc2412e805052d4bbf44cd6 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/e3a56bcf2cc2412e805052d4bbf44cd6 2024-11-15T17:42:40,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a38077616639487c9656d4456f0028e4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/a38077616639487c9656d4456f0028e4 2024-11-15T17:42:40,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1d56e16128784a2092aa097a78dde7e0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/1d56e16128784a2092aa097a78dde7e0 2024-11-15T17:42:40,116 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/recovered.edits/290.seqid, newMaxSeqId=290, maxSeqId=4 2024-11-15T17:42:40,116 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. 2024-11-15T17:42:40,116 DEBUG [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 248b48bc55c0fdcd5620a92777fad08f: Waiting for close lock at 1731692558378Running coprocessor pre-close hooks at 1731692558378Disabling compacts and flushes for region at 1731692558378Disabling writes for close at 1731692558379 (+1 ms)Obtaining lock to block concurrent updates at 1731692558379Preparing flush snapshotting stores in 248b48bc55c0fdcd5620a92777fad08f at 1731692558379Finished memstore snapshotting TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731692558380 (+1 ms)Flushing stores of TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f. at 1731692558381 (+1 ms)Flushing 248b48bc55c0fdcd5620a92777fad08f/A: creating writer at 1731692558381Flushing mob file 248b48bc55c0fdcd5620a92777fad08f/A: appending metadata at 1731692558389 (+8 ms)Flushing mob file 248b48bc55c0fdcd5620a92777fad08f/A: closing flushed file at 1731692558389Flushing 248b48bc55c0fdcd5620a92777fad08f/A: appending metadata at 1731692558804 (+415 ms)Flushing 248b48bc55c0fdcd5620a92777fad08f/A: closing flushed file at 1731692558804Flushing 248b48bc55c0fdcd5620a92777fad08f/B: creating writer at 1731692559219 (+415 ms)Flushing 248b48bc55c0fdcd5620a92777fad08f/B: appending metadata at 1731692559221 (+2 ms)Flushing 248b48bc55c0fdcd5620a92777fad08f/B: closing flushed file at 1731692559222 (+1 ms)Flushing 248b48bc55c0fdcd5620a92777fad08f/C: creating writer at 1731692559637 (+415 ms)Flushing 248b48bc55c0fdcd5620a92777fad08f/C: appending metadata at 1731692559640 (+3 ms)Flushing 248b48bc55c0fdcd5620a92777fad08f/C: closing flushed file at 1731692559640Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48689783: reopening flushed file at 1731692560048 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cff27c7: reopening flushed file at 1731692560054 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2643945: reopening flushed file at 1731692560058 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 248b48bc55c0fdcd5620a92777fad08f in 1683ms, sequenceid=287, compaction requested=true at 1731692560063 (+5 ms)Writing region close event to WAL at 1731692560113 (+50 ms)Running coprocessor post-close hooks at 1731692560116 (+3 ms)Closed at 1731692560116 2024-11-15T17:42:40,118 INFO [RS_CLOSE_REGION-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,118 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=248b48bc55c0fdcd5620a92777fad08f, regionState=CLOSED 2024-11-15T17:42:40,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 because future has completed 2024-11-15T17:42:40,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-15T17:42:40,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 248b48bc55c0fdcd5620a92777fad08f, server=4e3b17656000,45187,1731692383005 in 1.8950 sec 2024-11-15T17:42:40,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-15T17:42:40,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=248b48bc55c0fdcd5620a92777fad08f, UNASSIGN in 1.9000 sec 2024-11-15T17:42:40,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-15T17:42:40,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9040 sec 2024-11-15T17:42:40,128 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692560128"}]},"ts":"1731692560128"} 2024-11-15T17:42:40,130 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T17:42:40,130 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T17:42:40,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9210 sec 2024-11-15T17:42:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T17:42:40,367 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:40,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T17:42:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:40,371 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:42:40,373 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:40,378 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,381 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C, FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/recovered.edits] 2024-11-15T17:42:40,384 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/0a529e13645d456fb734eec09e66e76b to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/0a529e13645d456fb734eec09e66e76b 2024-11-15T17:42:40,385 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14fb676415e84239939e5310c8be69a0 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/14fb676415e84239939e5310c8be69a0 2024-11-15T17:42:40,386 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/6bd5684443c745d79a61efee07a67444 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/A/6bd5684443c745d79a61efee07a67444 2024-11-15T17:42:40,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/ab1485d8fb16430d93465fb11c97b653 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/ab1485d8fb16430d93465fb11c97b653 2024-11-15T17:42:40,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/e5d696addc9e43eeadd4a09d33fad4c4 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/e5d696addc9e43eeadd4a09d33fad4c4 2024-11-15T17:42:40,390 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/fc815a86fdcc4051a241de0d0f0c96e3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/B/fc815a86fdcc4051a241de0d0f0c96e3 2024-11-15T17:42:40,392 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/4df463b121624310a8814e025df86b40 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/4df463b121624310a8814e025df86b40 2024-11-15T17:42:40,393 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/70f1e5f939a54a4daa48db3375c08df3 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/70f1e5f939a54a4daa48db3375c08df3 2024-11-15T17:42:40,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/aa76a8e5217a40b3b7395f715fd8c6a7 to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/C/aa76a8e5217a40b3b7395f715fd8c6a7 2024-11-15T17:42:40,398 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/recovered.edits/290.seqid to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f/recovered.edits/290.seqid 2024-11-15T17:42:40,399 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/default/TestAcidGuarantees/248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,399 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T17:42:40,399 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:42:40,400 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T17:42:40,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e8062858ddc445bb5aa4769814cac6e_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e8062858ddc445bb5aa4769814cac6e_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152fcd1086fe304577af96b07e378b0828_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152fcd1086fe304577af96b07e378b0828_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,408 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115373bb10202a64ada8d23ae688e0d96cc_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115373bb10202a64ada8d23ae688e0d96cc_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,409 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560f3472e15c24139855d7a25b165b768_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111560f3472e15c24139855d7a25b165b768_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115687da4b06afb4f62ace56b3bc8753aba_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115687da4b06afb4f62ace56b3bc8753aba_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,413 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156f85ddb30f704eb08b583d29a39af88c_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156f85ddb30f704eb08b583d29a39af88c_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115854311aa15474b3faa559821f07e1983_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115854311aa15474b3faa559821f07e1983_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,415 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a357ed9c10e643e4bbe0b2306834ec61_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a357ed9c10e643e4bbe0b2306834ec61_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,417 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115accdd202159e4e8fb0d88153b8718a7c_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115accdd202159e4e8fb0d88153b8718a7c_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,418 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b94de083afa347f4b24c752cf00bbbfa_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b94de083afa347f4b24c752cf00bbbfa_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,419 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cb683325080b42b3a90b048af039e886_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cb683325080b42b3a90b048af039e886_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,420 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d228fefb90a3405789bca01ff85a0a40_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d228fefb90a3405789bca01ff85a0a40_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,422 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115da3101d69ef2430781b9e1ca9e756c0b_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115da3101d69ef2430781b9e1ca9e756c0b_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,423 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e9e603b9f247429e82599988af09db9d_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e9e603b9f247429e82599988af09db9d_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,424 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fe9aacc69ac441c0b3d09809d4b04ec0_248b48bc55c0fdcd5620a92777fad08f to hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fe9aacc69ac441c0b3d09809d4b04ec0_248b48bc55c0fdcd5620a92777fad08f 2024-11-15T17:42:40,425 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T17:42:40,427 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:40,430 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T17:42:40,432 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T17:42:40,433 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:40,433 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T17:42:40,433 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731692560433"}]},"ts":"9223372036854775807"} 2024-11-15T17:42:40,435 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T17:42:40,435 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 248b48bc55c0fdcd5620a92777fad08f, NAME => 'TestAcidGuarantees,,1731692532062.248b48bc55c0fdcd5620a92777fad08f.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T17:42:40,435 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T17:42:40,435 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731692560435"}]},"ts":"9223372036854775807"} 2024-11-15T17:42:40,437 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T17:42:40,437 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T17:42:40,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-15T17:42:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T17:42:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T17:42:40,477 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T17:42:40,491 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 237) - Thread LEAK? -, OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 359), ProcessCount=11 (was 11), AvailableMemoryMB=9126 (was 9388) 2024-11-15T17:42:40,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T17:42:40,491 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:42:40,491 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:42:40,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:40,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:40,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:42:40,491 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T17:42:40,491 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=164229068, stopped=false 2024-11-15T17:42:40,492 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4e3b17656000,41205,1731692382118 2024-11-15T17:42:40,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T17:42:40,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T17:42:40,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:42:40,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:42:40,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T17:42:40,557 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:42:40,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:42:40,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:42:40,557 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:42:40,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:40,559 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4e3b17656000,45187,1731692383005' ***** 2024-11-15T17:42:40,559 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T17:42:40,560 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T17:42:40,560 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T17:42:40,560 INFO [RS:0;4e3b17656000:45187 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(959): stopping server 4e3b17656000,45187,1731692383005 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4e3b17656000:45187. 2024-11-15T17:42:40,561 DEBUG [RS:0;4e3b17656000:45187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:42:40,561 DEBUG [RS:0;4e3b17656000:45187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T17:42:40,561 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T17:42:40,562 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T17:42:40,562 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T17:42:40,562 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:40,562 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T17:42:40,562 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T17:42:40,562 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T17:42:40,562 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T17:42:40,562 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T17:42:40,562 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T17:42:40,563 INFO [regionserver/4e3b17656000:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T17:42:40,580 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/info/71a5859efe1d4a4aaa753ef493a96ed7 is 95, key is TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474./info:/1731692413189/DeleteFamily/seqid=0 2024-11-15T17:42:40,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742470_1646 (size=6514) 2024-11-15T17:42:40,762 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:40,962 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:40,984 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/info/71a5859efe1d4a4aaa753ef493a96ed7 2024-11-15T17:42:41,011 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/ns/69603b9d2e4c466e9a558980c04156a6 is 93, key is TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474./ns:/1731692413141/DeleteFamily/seqid=0 2024-11-15T17:42:41,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742471_1647 (size=5906) 2024-11-15T17:42:41,163 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:41,190 INFO [regionserver/4e3b17656000:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T17:42:41,190 INFO [regionserver/4e3b17656000:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T17:42:41,236 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:42:41,363 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:41,415 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/ns/69603b9d2e4c466e9a558980c04156a6 2024-11-15T17:42:41,430 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/rep_barrier/ac9842dab08a454fba4dfd57f799ee29 is 102, key is TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474./rep_barrier:/1731692413141/DeleteFamily/seqid=0 2024-11-15T17:42:41,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742472_1648 (size=6025) 2024-11-15T17:42:41,563 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:41,764 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:41,835 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/rep_barrier/ac9842dab08a454fba4dfd57f799ee29 2024-11-15T17:42:41,861 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/table/a19030327eb946b685efa3a9b5a9071e is 96, key is TestAcidGuarantees,,1731692387767.ada8f074c978a899b609b6215cb2d474./table:/1731692413141/DeleteFamily/seqid=0 2024-11-15T17:42:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742473_1649 (size=5936) 2024-11-15T17:42:41,964 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:42,164 DEBUG [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T17:42:42,266 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/table/a19030327eb946b685efa3a9b5a9071e 2024-11-15T17:42:42,278 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/info/71a5859efe1d4a4aaa753ef493a96ed7 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/info/71a5859efe1d4a4aaa753ef493a96ed7 2024-11-15T17:42:42,284 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/info/71a5859efe1d4a4aaa753ef493a96ed7, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T17:42:42,286 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/ns/69603b9d2e4c466e9a558980c04156a6 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/ns/69603b9d2e4c466e9a558980c04156a6 2024-11-15T17:42:42,291 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/ns/69603b9d2e4c466e9a558980c04156a6, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T17:42:42,292 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/rep_barrier/ac9842dab08a454fba4dfd57f799ee29 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/rep_barrier/ac9842dab08a454fba4dfd57f799ee29 2024-11-15T17:42:42,297 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/rep_barrier/ac9842dab08a454fba4dfd57f799ee29, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T17:42:42,298 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/.tmp/table/a19030327eb946b685efa3a9b5a9071e as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/table/a19030327eb946b685efa3a9b5a9071e 2024-11-15T17:42:42,302 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/table/a19030327eb946b685efa3a9b5a9071e, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T17:42:42,302 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-15T17:42:42,307 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T17:42:42,308 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T17:42:42,308 INFO [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T17:42:42,308 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731692560562Running coprocessor pre-close hooks at 1731692560562Disabling compacts and flushes for region at 1731692560562Disabling writes for close at 1731692560562Obtaining lock to block concurrent updates at 1731692560562Preparing flush snapshotting stores in 1588230740 at 1731692560562Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731692560562Flushing stores of hbase:meta,,1.1588230740 at 1731692560563 (+1 ms)Flushing 1588230740/info: creating writer at 1731692560563Flushing 1588230740/info: appending metadata at 1731692560580 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731692560580Flushing 1588230740/ns: creating writer at 1731692560992 (+412 ms)Flushing 1588230740/ns: appending metadata at 1731692561011 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731692561011Flushing 1588230740/rep_barrier: creating writer at 1731692561418 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731692561430 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731692561430Flushing 1588230740/table: creating writer at 1731692561843 (+413 ms)Flushing 1588230740/table: appending metadata at 1731692561861 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731692561861Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@744e3166: reopening flushed file at 1731692562276 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d237041: reopening flushed file at 1731692562285 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27030352: reopening flushed file at 1731692562291 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14641edc: reopening flushed file at 1731692562297 (+6 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1731692562303 (+6 ms)Writing region close event to WAL at 1731692562304 (+1 ms)Running coprocessor post-close hooks at 1731692562308 (+4 ms)Closed at 1731692562308 2024-11-15T17:42:42,308 DEBUG [RS_CLOSE_META-regionserver/4e3b17656000:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T17:42:42,365 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(976): stopping server 4e3b17656000,45187,1731692383005; all regions closed. 2024-11-15T17:42:42,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741834_1010 (size=25344) 2024-11-15T17:42:42,374 DEBUG [RS:0;4e3b17656000:45187 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/oldWALs 2024-11-15T17:42:42,375 INFO [RS:0;4e3b17656000:45187 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4e3b17656000%2C45187%2C1731692383005.meta:.meta(num 1731692387312) 2024-11-15T17:42:42,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741832_1008 (size=14610308) 2024-11-15T17:42:42,379 DEBUG [RS:0;4e3b17656000:45187 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/oldWALs 2024-11-15T17:42:42,379 INFO [RS:0;4e3b17656000:45187 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4e3b17656000%2C45187%2C1731692383005:(num 1731692386300) 2024-11-15T17:42:42,379 DEBUG [RS:0;4e3b17656000:45187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:42:42,379 INFO [RS:0;4e3b17656000:45187 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T17:42:42,379 INFO [RS:0;4e3b17656000:45187 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T17:42:42,379 INFO [RS:0;4e3b17656000:45187 {}] hbase.ChoreService(370): Chore service for: regionserver/4e3b17656000:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T17:42:42,380 INFO [RS:0;4e3b17656000:45187 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T17:42:42,380 INFO [regionserver/4e3b17656000:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T17:42:42,380 INFO [RS:0;4e3b17656000:45187 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45187 2024-11-15T17:42:42,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T17:42:42,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4e3b17656000,45187,1731692383005 2024-11-15T17:42:42,432 INFO [RS:0;4e3b17656000:45187 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T17:42:42,434 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4e3b17656000,45187,1731692383005] 2024-11-15T17:42:42,454 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4e3b17656000,45187,1731692383005 already deleted, retry=false 2024-11-15T17:42:42,454 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4e3b17656000,45187,1731692383005 expired; onlineServers=0 2024-11-15T17:42:42,454 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4e3b17656000,41205,1731692382118' ***** 2024-11-15T17:42:42,454 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T17:42:42,455 INFO [M:0;4e3b17656000:41205 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T17:42:42,455 INFO [M:0;4e3b17656000:41205 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T17:42:42,455 DEBUG [M:0;4e3b17656000:41205 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T17:42:42,455 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T17:42:42,455 DEBUG [M:0;4e3b17656000:41205 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T17:42:42,455 DEBUG [master/4e3b17656000:0:becomeActiveMaster-HFileCleaner.small.0-1731692385981 {}] cleaner.HFileCleaner(306): Exit Thread[master/4e3b17656000:0:becomeActiveMaster-HFileCleaner.small.0-1731692385981,5,FailOnTimeoutGroup] 2024-11-15T17:42:42,455 DEBUG [master/4e3b17656000:0:becomeActiveMaster-HFileCleaner.large.0-1731692385980 {}] cleaner.HFileCleaner(306): Exit Thread[master/4e3b17656000:0:becomeActiveMaster-HFileCleaner.large.0-1731692385980,5,FailOnTimeoutGroup] 2024-11-15T17:42:42,456 INFO [M:0;4e3b17656000:41205 {}] hbase.ChoreService(370): Chore service for: master/4e3b17656000:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T17:42:42,456 INFO [M:0;4e3b17656000:41205 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T17:42:42,456 DEBUG [M:0;4e3b17656000:41205 {}] master.HMaster(1795): Stopping service threads 2024-11-15T17:42:42,456 INFO [M:0;4e3b17656000:41205 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T17:42:42,457 INFO [M:0;4e3b17656000:41205 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T17:42:42,457 ERROR [M:0;4e3b17656000:41205 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:38333 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38333,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T17:42:42,458 INFO [M:0;4e3b17656000:41205 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T17:42:42,458 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T17:42:42,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T17:42:42,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:42:42,465 DEBUG [M:0;4e3b17656000:41205 {}] zookeeper.ZKUtil(347): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T17:42:42,465 WARN [M:0;4e3b17656000:41205 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T17:42:42,466 INFO [M:0;4e3b17656000:41205 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/.lastflushedseqids 2024-11-15T17:42:42,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742474_1650 (size=72) 2024-11-15T17:42:42,540 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T17:42:42,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T17:42:42,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T17:42:42,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:42:42,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x10140de572c0001, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:42:42,547 INFO [RS:0;4e3b17656000:45187 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T17:42:42,547 INFO [RS:0;4e3b17656000:45187 {}] regionserver.HRegionServer(1031): Exiting; stopping=4e3b17656000,45187,1731692383005; zookeeper connection closed. 2024-11-15T17:42:42,547 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@25e39517 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@25e39517 2024-11-15T17:42:42,548 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T17:42:42,878 INFO [M:0;4e3b17656000:41205 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T17:42:42,878 INFO [M:0;4e3b17656000:41205 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T17:42:42,878 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T17:42:42,878 INFO [M:0;4e3b17656000:41205 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:42:42,879 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:42:42,879 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T17:42:42,879 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:42:42,879 INFO [M:0;4e3b17656000:41205 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=753.58 KB heapSize=925.89 KB 2024-11-15T17:42:42,900 DEBUG [M:0;4e3b17656000:41205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/876613834e334596a9ad34eb8c3fb1bd is 82, key is hbase:meta,,1/info:regioninfo/1731692387388/Put/seqid=0 2024-11-15T17:42:42,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742475_1651 (size=5672) 2024-11-15T17:42:43,303 INFO [M:0;4e3b17656000:41205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2127 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/876613834e334596a9ad34eb8c3fb1bd 2024-11-15T17:42:43,322 DEBUG [M:0;4e3b17656000:41205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69329dddfa534835a5f7495df8e5a758 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1731692535168/Put/seqid=0 2024-11-15T17:42:43,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742476_1652 (size=43849) 2024-11-15T17:42:43,726 INFO [M:0;4e3b17656000:41205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=753.02 KB at sequenceid=2127 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69329dddfa534835a5f7495df8e5a758 2024-11-15T17:42:43,737 INFO [M:0;4e3b17656000:41205 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 69329dddfa534835a5f7495df8e5a758 2024-11-15T17:42:43,751 DEBUG [M:0;4e3b17656000:41205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/857832ba38bd4d14b14c1da8a8273248 is 69, key is 4e3b17656000,45187,1731692383005/rs:state/1731692386037/Put/seqid=0 2024-11-15T17:42:43,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073742477_1653 (size=5156) 2024-11-15T17:42:44,154 INFO [M:0;4e3b17656000:41205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2127 (bloomFilter=true), to=hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/857832ba38bd4d14b14c1da8a8273248 2024-11-15T17:42:44,160 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/876613834e334596a9ad34eb8c3fb1bd as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/876613834e334596a9ad34eb8c3fb1bd 2024-11-15T17:42:44,165 INFO [M:0;4e3b17656000:41205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/876613834e334596a9ad34eb8c3fb1bd, entries=8, sequenceid=2127, filesize=5.5 K 2024-11-15T17:42:44,166 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/69329dddfa534835a5f7495df8e5a758 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/69329dddfa534835a5f7495df8e5a758 2024-11-15T17:42:44,170 INFO [M:0;4e3b17656000:41205 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 69329dddfa534835a5f7495df8e5a758 2024-11-15T17:42:44,170 INFO [M:0;4e3b17656000:41205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/69329dddfa534835a5f7495df8e5a758, entries=168, sequenceid=2127, filesize=42.8 K 2024-11-15T17:42:44,171 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/857832ba38bd4d14b14c1da8a8273248 as hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/857832ba38bd4d14b14c1da8a8273248 2024-11-15T17:42:44,175 INFO [M:0;4e3b17656000:41205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38333/user/jenkins/test-data/4d9f8c41-c8b5-d9ca-62aa-f44cd1a0f46e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/857832ba38bd4d14b14c1da8a8273248, entries=1, sequenceid=2127, filesize=5.0 K 2024-11-15T17:42:44,176 INFO [M:0;4e3b17656000:41205 {}] regionserver.HRegion(3140): Finished flush of dataSize ~753.58 KB/771666, heapSize ~925.59 KB/947808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2127, compaction requested=false 2024-11-15T17:42:44,178 INFO [M:0;4e3b17656000:41205 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:42:44,178 DEBUG [M:0;4e3b17656000:41205 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731692562878Disabling compacts and flushes for region at 1731692562878Disabling writes for close at 1731692562879 (+1 ms)Obtaining lock to block concurrent updates at 1731692562879Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731692562879Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=771666, getHeapSize=948048, getOffHeapSize=0, getCellsCount=2320 at 1731692562880 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731692562881 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731692562881Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731692562900 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731692562900Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731692563306 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731692563321 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731692563321Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731692563737 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731692563751 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731692563751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71271fd0: reopening flushed file at 1731692564159 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@989b882: reopening flushed file at 1731692564165 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b2723d7: reopening flushed file at 1731692564170 (+5 ms)Finished flush of dataSize ~753.58 KB/771666, heapSize ~925.59 KB/947808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2127, compaction requested=false at 1731692564176 (+6 ms)Writing region close event to WAL at 1731692564178 (+2 ms)Closed at 1731692564178 2024-11-15T17:42:44,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43115 is added to blk_1073741830_1006 (size=911112) 2024-11-15T17:42:44,181 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T17:42:44,181 INFO [M:0;4e3b17656000:41205 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T17:42:44,181 INFO [M:0;4e3b17656000:41205 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41205 2024-11-15T17:42:44,182 INFO [M:0;4e3b17656000:41205 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T17:42:44,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:42:44,358 INFO [M:0;4e3b17656000:41205 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T17:42:44,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41205-0x10140de572c0000, quorum=127.0.0.1:61131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:42:44,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T17:42:44,368 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T17:42:44,369 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T17:42:44,369 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T17:42:44,369 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/hadoop.log.dir/,STOPPED} 2024-11-15T17:42:44,371 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T17:42:44,371 WARN [BP-624044830-172.17.0.2-1731692378459 heartbeating to localhost/127.0.0.1:38333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T17:42:44,371 WARN [BP-624044830-172.17.0.2-1731692378459 heartbeating to localhost/127.0.0.1:38333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-624044830-172.17.0.2-1731692378459 (Datanode Uuid cdf78b8e-f28b-455a-8346-40004e1fb9d6) service to localhost/127.0.0.1:38333 2024-11-15T17:42:44,371 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T17:42:44,374 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/data/data1/current/BP-624044830-172.17.0.2-1731692378459 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T17:42:44,374 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/cluster_2b1a8450-fbeb-9159-d463-a4b4ea84404e/data/data2/current/BP-624044830-172.17.0.2-1731692378459 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T17:42:44,375 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T17:42:44,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T17:42:44,382 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T17:42:44,382 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T17:42:44,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T17:42:44,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf6ae6cc-cd7d-9894-c3a8-b3e04bae18ff/hadoop.log.dir/,STOPPED} 2024-11-15T17:42:44,397 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T17:42:44,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down